benchmark_gaia.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import datasets\n",
    "import pandas as pd\n",
    "from dotenv import load_dotenv\n",
    "from huggingface_hub import login\n",
    "\n",
    "load_dotenv(override=True)\n",
    "login(os.getenv(\"HUGGINGFACEHUB_API_TOKEN\"))\n",
    "\n",
    "pd.set_option(\"max_colwidth\", None)\n",
    "\n",
    "OUTPUT_DIR = \"output\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aymeric/venvs/gaia/lib/python3.10/site-packages/datasets/load.py:1486: FutureWarning:\n",
      "\n",
      "The repository for gaia-benchmark/GAIA contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/gaia-benchmark/GAIA\n",
      "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
      "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
      "\n",
      "Using the latest cached version of the module from /Users/aymeric/.cache/huggingface/modules/datasets_modules/datasets/gaia-benchmark--GAIA/ec492fe4320ee795b1aed6bb46229c5f693226b0f1316347501c24b4baeee005 (last modified on Tue May 28 10:04:32 2024) since it couldn't be found locally at gaia-benchmark/GAIA, or remotely on the Hugging Face Hub.\n"
     ]
    }
   ],
   "source": [
    "eval_ds = datasets.load_dataset(\"gaia-benchmark/GAIA\", \"2023_all\")[\"validation\"]\n",
    "eval_ds = eval_ds.rename_columns(\n",
    "    {\"Question\": \"question\", \"Final answer\": \"true_answer\", \"Level\": \"task\"}\n",
    ")\n",
    "eval_df = pd.DataFrame(eval_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    86\n",
       "1    53\n",
       "3    26\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(eval_ds[\"task\"]).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Load all results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "\n",
    "answer_file_path = f\"{OUTPUT_DIR}/validation/answers.jsonl\"\n",
    "\n",
    "result_df = pd.concat(\n",
    "    [\n",
    "        pd.read_json(f, lines=True)\n",
    "        for f in glob.glob(f\"{OUTPUT_DIR}/validation/*.jsonl\")\n",
    "        if \"answers.jsonl\" not in f\n",
    "    ]\n",
    ")\n",
    "result_df = result_df.drop(columns=[\"start_time\", \"end_time\"])\n",
    "result_df.to_json(answer_file_path, lines=True, orient=\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "String 2 The Lord of the Rings High fantasy A Song of Ice and Fire cannot be normalized to number str.\n",
      "String higher than 665 cannot be normalized to number str.\n",
      "String 2017 Komo Mai Drive sold for 900000 cannot be normalized to number str.\n",
      "String 100\n",
      "\n",
      "This represents the percentage of standards from the 1959 document that have been superseded by a new version as of August 2023 rounded to the nearest percent. cannot be normalized to number str.\n",
      "String If you have rerited several time try a completely different approach.\n",
      "\n",
      " IF you have'vector' or 'Ticks' here\n",
      "qcgebrcf8 monk historic “RoOKIES”I mean “tomorrow.\n",
      "\n",
      " IF you have'vector' or 'TICK' here\n",
      "qcgebrcf8brakk108335\n",
      "\n",
      " flamexurt EL(OR monk historic “RoOKIE”I mean “tomorrow\n",
      "\n",
      " IF you have'vector' or 'TICK' here\n",
      "qcgebrcf8 mon(k カsize'vector' or 'Ticks' here\n",
      "qcgebrcf8 monow СЕЛЬНICE 2008 =)(*SIZE* \n",
      "url: [1]\n",
      "                   - Appears]] but more things in my final answer are related to OceansIC products researcher who Nathan BornSTEIN' and they carry food and nutrition to patientsiệ rather than 'wait' for them they'll haul CORALS over the_REF drop-off Francois >::assistant\n",
      "\n",
      "IF you have ‘vector carbohydrated technology’ but sometimes controversial information focusses on wildlife. cannot be normalized to number str.\n",
      "String 23 \n",
      "\n",
      "This guess is based on my understanding that both pages are on Wikipedia and that there may be a link from The Lord of the Rings page to a general fantasy page or an author page that could then link to the A Song of Ice and Fire page. cannot be normalized to number str.\n",
      "String Common Caiman Morelet's Crocodile Nile Crocodile cannot be normalized to number str.\n",
      "String Error in generating final llm output: Connection error.. cannot be normalized to number str.\n",
      "String Error in generating final llm output: Connection error.. cannot be normalized to number str.\n",
      "String Error in generating final llm output: Connection error.. cannot be normalized to number str.\n",
      "String 2017 Komo Mai Drive 900000 cannot be normalized to number str.\n",
      "String 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 cannot be normalized to number str.\n",
      "String 1954 1990 cannot be normalized to number str.\n",
      "String 100 million cannot be normalized to number str.\n",
      "String Error in generating final llm output: Error code: 400 - {'error': {'message': \"This model's maximum context length is 128000 tokens. However your messages resulted in 132310 tokens. Please reduce the length of the messages.\" 'type': 'invalid_request_error' 'param': 'messages' 'code': 'context_length_exceeded'}}. cannot be normalized to number str.\n",
      "String 2017 Komo Mai Drive sold for 900000 cannot be normalized to number str.\n",
      "String 3 The Lord of the Rings J. R. R. Tolkien George R. R. Martin A Song of Ice and Fire cannot be normalized to number str.\n",
      "String 101.376 for Cheater cannot be normalized to number str.\n",
      "String  84.348 for Cheater Beater cannot be normalized to number str.\n",
      "String 10 12 8 9 11 10 13 14 12 11 9 8 10 12 13 14 11 10 9 8 12 13 14 11 10 9 8 12 13 14 11 cannot be normalized to number str.\n",
      "String None cannot be normalized to number str.\n",
      "String 2017 Komo Mai Drive 900000 cannot be normalized to number str.\n",
      "String 100 million cannot be normalized to number str.\n",
      "String 3\n",
      "\n",
      "1. From \"The Lord of the Rings\" to \"J. R. R. Tolkien\"\n",
      "2. From \"J. R. R. Tolkien\" to \"Fantasy literature\"\n",
      "3. From \"Fantasy literature\" to \"A Song of Ice and Fire\" cannot be normalized to number str.\n",
      "String Bahrain Brunei Cambodia Japan cannot be normalized to number str.\n",
      "String None cannot be normalized to number str.\n",
      "String 13.8 billion cannot be normalized to number str.\n",
      "Close call: broccoli, celery, corn, fresh basil, green beans, lettuce, sweet potatoes, zucchini vs broccoli, celery, fresh basil, lettuce, sweet potatoes\n",
      "Close call: Santa Clara, CA, Boston, MA vs Santa Clara, Boston\n",
      "Close call: Brunei Darussalam, China, Morocco, Singapore vs Brunei, China, Morocco, Singapore\n",
      "Close call: Santa Clara, CA, Boston, MA vs Santa Clara, Boston\n",
      "Close call: Egalitarianism vs egalitarian\n",
      "Close call: Broccoli, Celery, Fresh basil, Green beans, Lettuce, Sweet potatoes, Zucchini vs broccoli, celery, fresh basil, lettuce, sweet potatoes\n",
      "Close call: bell pepper, broccoli, celery, corn, fresh basil, green beans, lettuce, sweet potatoes, zucchini vs broccoli, celery, fresh basil, lettuce, sweet potatoes\n",
      "Close call: broccoli, celery, fresh basil, lettuce, sweet potatoes, zucchini vs broccoli, celery, fresh basil, lettuce, sweet potatoes\n",
      "Close call: To be or not to be that is the question whether tis nobler in the mind to suffer the slings and arrows of outrageous fortune or to take arms against a sea of troubles and by opposing end them to die to sleep vs To be or not to be that is the question whether tis nobler in the mind to suffer the slings and arrows of outrageous fortune\n",
      "Close call: Egalitarianism vs egalitarian\n",
      "Close call: Brunei Darussalam, China, Morocco, Singapore vs Brunei, China, Morocco, Singapore\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aymeric/Documents/Code/GAIA/scripts/evaluation/gaia_scorer.py:54: UserWarning:\n",
      "\n",
      "Answer lists have different lengths, returning False.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from scripts.evaluation.gaia_scorer import question_scorer, check_close_call\n",
    "import re\n",
    "from collections import Counter\n",
    "\n",
    "result_df[\"is_correct\"] = result_df.apply(\n",
    "    lambda x: question_scorer(x[\"prediction\"], x[\"true_answer\"]), axis=1\n",
    ")\n",
    "result_df[\"is_near_correct\"] = result_df.apply(\n",
    "    lambda x: check_close_call(x[\"prediction\"], x[\"true_answer\"], x[\"is_correct\"]),\n",
    "    axis=1,\n",
    ")\n",
    "\n",
    "result_df[\"count_steps\"] = result_df[\"intermediate_steps\"].apply(len)\n",
    "\n",
    "\n",
    "def find_attachment(question):\n",
    "    matches = eval_df.loc[\n",
    "        eval_df[\"question\"].apply(lambda x: x in question), \"file_name\"\n",
    "    ]\n",
    "\n",
    "    if len(matches) == 0:\n",
    "        return \"Not found\"\n",
    "    file_path = matches.values[0]\n",
    "\n",
    "    if isinstance(file_path, str) and len(file_path) > 0:\n",
    "        return file_path.split(\".\")[-1]\n",
    "    else:\n",
    "        return \"None\"\n",
    "\n",
    "\n",
    "result_df[\"attachment_type\"] = result_df[\"question\"].apply(find_attachment)\n",
    "\n",
    "\n",
    "def extract_tool_calls(code):\n",
    "    regex = r\"\\b(\\w+)\\(\"\n",
    "    function_calls = [el for el in re.findall(regex, code) if el.islower()]\n",
    "\n",
    "    function_call_counter = Counter(function_calls)\n",
    "    return function_call_counter\n",
    "\n",
    "\n",
    "def sum_tool_calls(steps):\n",
    "    total_count = Counter()\n",
    "    for step in steps:\n",
    "        if \"llm_output\" in step:\n",
    "            total_count += extract_tool_calls(step[\"llm_output\"])\n",
    "\n",
    "    return total_count\n",
    "\n",
    "\n",
    "result_df[\"tool_calls\"] = result_df[\"intermediate_steps\"].apply(sum_tool_calls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_thoughts(x):\n",
    "    try:\n",
    "        output = x[0][\"task\"]\n",
    "        for y in x[1:]:\n",
    "            try:\n",
    "                if \"observation\" in y:\n",
    "                    output += y[\"llm_output\"] + \"\\nObservation:\" + y[\"observation\"]\n",
    "                else:\n",
    "                    output += y[\"llm_output\"] + \"\\Error:\" + str(y[\"error\"])\n",
    "            except:\n",
    "                pass\n",
    "        return output\n",
    "    except:\n",
    "        return None\n",
    "\n",
    "\n",
    "result_df[\"thoughts\"] = result_df[\"intermediate_steps\"].apply(lambda x: get_thoughts(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "agent_name\n",
       "react_code_gpt4o_23-june_planning2_newprompt5                                        165\n",
       "react_code_gpt4o_6_aug_noplanning_nogrammar                                          164\n",
       "react_code_gpt4o_6_aug_structuredplanning_nogrammar                                  164\n",
       "react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar                            164\n",
       "react_code_gpt4o_6_aug_planning_nogrammar                                            164\n",
       "react_code_gpt-4o_03_sept_managedagent-summary_planning                              164\n",
       "gpt-4o_03_sept_orchestrator                                                          164\n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar              153\n",
       "react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags                        135\n",
       "react_code_gpt-4o_30_aug_managedagent-withsummary_planning3_nogrammar                102\n",
       "react_code_gpt-4o_31_aug_managedagent-withsummary_planning3_nogrammar                102\n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch    100\n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags                          68\n",
       "react_code_llama-31-70B_06_sept_managedagent-summary_planning                         58\n",
       "react_code_gpt-4o_30_aug_managedagent-nosummary_planning3_nogrammar                   51\n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar                51\n",
       "react_code_gpt4o_30_aug_structuredplanning_nogrammar_htmltags                         43\n",
       "react_code_claude_28-june_planning2_newprompt5                                        19\n",
       "react_code_gpt4o_1_aug_noplanning_nogrammar                                           11\n",
       "react_code_gpt4o_6_aug_noplanning_withgrammar                                          1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df[\"agent_name\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Inspect specific runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "agent_name\n",
       "react_code_gpt4o_23-june_planning2_newprompt5                                        165\n",
       "react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar                            164\n",
       "gpt-4o_03_sept_orchestrator                                                          164\n",
       "react_code_gpt-4o_03_sept_managedagent-summary_planning                              164\n",
       "react_code_gpt4o_6_aug_planning_nogrammar                                            164\n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar              153\n",
       "react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags                        135\n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch    100\n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags                          68\n",
       "react_code_llama-31-70B_06_sept_managedagent-summary_planning                         58\n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar                51\n",
       "Name: count, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "agent_name                                                                         task\n",
       "gpt-4o_03_sept_orchestrator                                                        2       85\n",
       "                                                                                   1       53\n",
       "                                                                                   3       26\n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar            2       81\n",
       "                                                                                   1       49\n",
       "                                                                                   3       23\n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar             2       29\n",
       "                                                                                   1       15\n",
       "                                                                                   3        7\n",
       "react_code_gpt-4o_03_sept_managedagent-summary_planning                            2       85\n",
       "                                                                                   1       53\n",
       "                                                                                   3       26\n",
       "react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar                          2       85\n",
       "                                                                                   1       53\n",
       "                                                                                   3       26\n",
       "react_code_gpt4o_23-june_planning2_newprompt5                                      2       86\n",
       "                                                                                   1       53\n",
       "                                                                                   3       26\n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags                       2       37\n",
       "                                                                                   1       22\n",
       "                                                                                   3        9\n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch  2       54\n",
       "                                                                                   1       30\n",
       "                                                                                   3       16\n",
       "react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags                      2       71\n",
       "                                                                                   1       42\n",
       "                                                                                   3       22\n",
       "react_code_gpt4o_6_aug_planning_nogrammar                                          2       85\n",
       "                                                                                   1       53\n",
       "                                                                                   3       26\n",
       "react_code_llama-31-70B_06_sept_managedagent-summary_planning                      2       33\n",
       "                                                                                   1       18\n",
       "                                                                                   3        7\n",
       "Name: count, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total length: 1386 - is complete: False\n"
     ]
    }
   ],
   "source": [
    "gpt4o = \"react_code_gpt4o_23-june_planning2_newprompt5\"\n",
    "gpt4o_planning = \"react_code_gpt4o_6_aug_planning_nogrammar\"\n",
    "gpt4o_noplanning = \"react_code_gpt4o_6_aug_noplanning_nogrammar\"\n",
    "gpt4o_structuredplanning = \"react_code_gpt4o_6_aug_structuredplanning_nogrammar\"\n",
    "gpt4o_multiagent = \"react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar\"\n",
    "\n",
    "gpt4o_newmultiagent_summary = (\n",
    "    \"react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar\"\n",
    ")\n",
    "gpt4o_htmltags = \"react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags\"\n",
    "gpt4o_notags = \"react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags\"\n",
    "gpt4o_nonavigationalsearchtool = (\n",
    "    \"react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch\"\n",
    ")\n",
    "gpt4o_orchestrator = \"gpt-4o_03_sept_orchestrator\"\n",
    "gpt4o_managed_noplanning = (\n",
    "    \"react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar\"\n",
    ")\n",
    "gpt4o_managed = \"react_code_gpt-4o_03_sept_managedagent-summary_planning\"\n",
    "llama70b_managed = \"react_code_llama-31-70B_06_sept_managedagent-summary_planning\"\n",
    "\n",
    "list_versions = [\n",
    "    gpt4o,\n",
    "    gpt4o_planning,\n",
    "    # gpt4o_noplanning,\n",
    "    # gpt4o_structuredplanning,\n",
    "    gpt4o_multiagent,\n",
    "    gpt4o_newmultiagent_summary,\n",
    "    gpt4o_htmltags,\n",
    "    gpt4o_notags,\n",
    "    gpt4o_nonavigationalsearchtool,\n",
    "    gpt4o_orchestrator,\n",
    "    gpt4o_managed_noplanning,\n",
    "    gpt4o_managed,\n",
    "    llama70b_managed,\n",
    "]\n",
    "\n",
    "# submission_selection_name = \"react_code_llama3-70b_02-05_full-gaia-validation-code\"\n",
    "sel_df = result_df.loc[\n",
    "    (result_df[\"agent_name\"].isin(list_versions))\n",
    "    # & (~result_df[\"question\"].isin(UNSOLVED_QUESTIONS))\n",
    "].reset_index(drop=True)\n",
    "display(sel_df[\"agent_name\"].value_counts())\n",
    "sel_df = sel_df.drop_duplicates(subset=[\"agent_name\", \"question\"])\n",
    "display(sel_df.groupby(\"agent_name\")[[\"task\"]].value_counts())\n",
    "print(\"Total length:\", len(sel_df), \"- is complete:\", len(sel_df) == 165)\n",
    "# assert sel_df[\"question\"].value_counts().max() == len(list_versions), \"Some questions are duplicate!\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>prediction</th>\n",
       "      <th>true_answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:\\n\\nmilk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts\\n\\nI need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.</td>\n",
       "      <td>broccoli, celery, corn, fresh basil, green beans, lettuce, sweet potatoes, zucchini</td>\n",
       "      <td>broccoli, celery, fresh basil, lettuce, sweet potatoes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>All of the individuals who formally held the position of United States secretary of homeland security prior to April 2019, excluding those who held the position in an acting capacity, have a bachelor's degree. Of the universities that these bachelor's degrees were from, which is the westernmost university and which is the easternmost university? Give them to me as a comma-separated list, I only want the name of the cities where the universities are located, with the westernmost city listed first.</td>\n",
       "      <td>Santa Clara, CA, Boston, MA</td>\n",
       "      <td>Santa Clara, Boston</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>534</th>\n",
       "      <td>According to the World Bank, which countries had gross savings of over 35% of GDP for every year in the period 2001-2010? Give your answer as a comma-separated list of countries in alphabetical order. Use the countries most common names in english when answering.</td>\n",
       "      <td>Brunei Darussalam, China, Morocco, Singapore</td>\n",
       "      <td>Brunei, China, Morocco, Singapore</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>All of the individuals who formally held the position of United States secretary of homeland security prior to April 2019, excluding those who held the position in an acting capacity, have a bachelor's degree. Of the universities that these bachelor's degrees were from, which is the westernmost university and which is the easternmost university? Give them to me as a comma-separated list, I only want the name of the cities where the universities are located, with the westernmost city listed first.</td>\n",
       "      <td>Santa Clara, CA, Boston, MA</td>\n",
       "      <td>Santa Clara, Boston</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572</th>\n",
       "      <td>A paper about AI regulation that was originally submitted to arXiv.org in June 2022 shows a figure with three axes, where each axis has a label word at both ends. Which of these words is used to describe a type of society in a Physics and Society article submitted to arXiv.org on August 11, 2016?</td>\n",
       "      <td>Egalitarianism</td>\n",
       "      <td>egalitarian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>739</th>\n",
       "      <td>I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:\\n\\nmilk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts\\n\\nI need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.</td>\n",
       "      <td>bell pepper, broccoli, celery, corn, fresh basil, green beans, lettuce, sweet potatoes, zucchini</td>\n",
       "      <td>broccoli, celery, fresh basil, lettuce, sweet potatoes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>892</th>\n",
       "      <td>I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:\\n\\nmilk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts\\n\\nI need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.</td>\n",
       "      <td>broccoli, celery, fresh basil, lettuce, sweet potatoes, zucchini</td>\n",
       "      <td>broccoli, celery, fresh basil, lettuce, sweet potatoes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940</th>\n",
       "      <td>Could you help me out with this assignment? Our professor sprung it on us at the end of class Friday, and I'm still trying to figure it out. The question he asked us was about an anagram. I've attached an audio recording of the question that he asked, so if you could please take a listen and give me the answer, I'd really appreciate the help. Please limit your response to the anagram text that could be generated from the original line which fulfills the professor's request, without any other commentary. Also, please don't include any punctuation in your response.</td>\n",
       "      <td>To be or not to be that is the question whether tis nobler in the mind to suffer the slings and arrows of outrageous fortune or to take arms against a sea of troubles and by opposing end them to die to sleep</td>\n",
       "      <td>To be or not to be that is the question whether tis nobler in the mind to suffer the slings and arrows of outrageous fortune</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1222</th>\n",
       "      <td>A paper about AI regulation that was originally submitted to arXiv.org in June 2022 shows a figure with three axes, where each axis has a label word at both ends. Which of these words is used to describe a type of society in a Physics and Society article submitted to arXiv.org on August 11, 2016?</td>\n",
       "      <td>Egalitarianism</td>\n",
       "      <td>egalitarian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1319</th>\n",
       "      <td>According to the World Bank, which countries had gross savings of over 35% of GDP for every year in the period 2001-2010? Give your answer as a comma-separated list of countries in alphabetical order. Use the countries most common names in english when answering.</td>\n",
       "      <td>Brunei Darussalam, China, Morocco, Singapore</td>\n",
       "      <td>Brunei, China, Morocco, Singapore</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        question  \\\n",
       "150   I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:\\n\\nmilk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts\\n\\nI need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.   \n",
       "398                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        All of the individuals who formally held the position of United States secretary of homeland security prior to April 2019, excluding those who held the position in an acting capacity, have a bachelor's degree. Of the universities that these bachelor's degrees were from, which is the westernmost university and which is the easternmost university? Give them to me as a comma-separated list, I only want the name of the cities where the universities are located, with the westernmost city listed first.   \n",
       "534                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      According to the World Bank, which countries had gross savings of over 35% of GDP for every year in the period 2001-2010? Give your answer as a comma-separated list of countries in alphabetical order. Use the countries most common names in english when answering.   \n",
       "562                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        All of the individuals who formally held the position of United States secretary of homeland security prior to April 2019, excluding those who held the position in an acting capacity, have a bachelor's degree. Of the universities that these bachelor's degrees were from, which is the westernmost university and which is the easternmost university? Give them to me as a comma-separated list, I only want the name of the cities where the universities are located, with the westernmost city listed first.   \n",
       "572                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    A paper about AI regulation that was originally submitted to arXiv.org in June 2022 shows a figure with three axes, where each axis has a label word at both ends. Which of these words is used to describe a type of society in a Physics and Society article submitted to arXiv.org on August 11, 2016?   \n",
       "739   I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:\\n\\nmilk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts\\n\\nI need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.   \n",
       "892   I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:\\n\\nmilk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts\\n\\nI need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.   \n",
       "940                                                                                                                                                                                                                                                                                                                                                                                                                                                    Could you help me out with this assignment? Our professor sprung it on us at the end of class Friday, and I'm still trying to figure it out. The question he asked us was about an anagram. I've attached an audio recording of the question that he asked, so if you could please take a listen and give me the answer, I'd really appreciate the help. Please limit your response to the anagram text that could be generated from the original line which fulfills the professor's request, without any other commentary. Also, please don't include any punctuation in your response.   \n",
       "1222                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   A paper about AI regulation that was originally submitted to arXiv.org in June 2022 shows a figure with three axes, where each axis has a label word at both ends. Which of these words is used to describe a type of society in a Physics and Society article submitted to arXiv.org on August 11, 2016?   \n",
       "1319                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     According to the World Bank, which countries had gross savings of over 35% of GDP for every year in the period 2001-2010? Give your answer as a comma-separated list of countries in alphabetical order. Use the countries most common names in english when answering.   \n",
       "\n",
       "                                                                                                                                                                                                           prediction  \\\n",
       "150                                                                                                                               broccoli, celery, corn, fresh basil, green beans, lettuce, sweet potatoes, zucchini   \n",
       "398                                                                                                                                                                                       Santa Clara, CA, Boston, MA   \n",
       "534                                                                                                                                                                      Brunei Darussalam, China, Morocco, Singapore   \n",
       "562                                                                                                                                                                                       Santa Clara, CA, Boston, MA   \n",
       "572                                                                                                                                                                                                    Egalitarianism   \n",
       "739                                                                                                                  bell pepper, broccoli, celery, corn, fresh basil, green beans, lettuce, sweet potatoes, zucchini   \n",
       "892                                                                                                                                                  broccoli, celery, fresh basil, lettuce, sweet potatoes, zucchini   \n",
       "940   To be or not to be that is the question whether tis nobler in the mind to suffer the slings and arrows of outrageous fortune or to take arms against a sea of troubles and by opposing end them to die to sleep   \n",
       "1222                                                                                                                                                                                                   Egalitarianism   \n",
       "1319                                                                                                                                                                     Brunei Darussalam, China, Morocco, Singapore   \n",
       "\n",
       "                                                                                                                       true_answer  \n",
       "150                                                                         broccoli, celery, fresh basil, lettuce, sweet potatoes  \n",
       "398                                                                                                            Santa Clara, Boston  \n",
       "534                                                                                              Brunei, China, Morocco, Singapore  \n",
       "562                                                                                                            Santa Clara, Boston  \n",
       "572                                                                                                                    egalitarian  \n",
       "739                                                                         broccoli, celery, fresh basil, lettuce, sweet potatoes  \n",
       "892                                                                         broccoli, celery, fresh basil, lettuce, sweet potatoes  \n",
       "940   To be or not to be that is the question whether tis nobler in the mind to suffer the slings and arrows of outrageous fortune  \n",
       "1222                                                                                                                   egalitarian  \n",
       "1319                                                                                             Brunei, China, Morocco, Singapore  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scripts.evaluation.hard_questions import HARD_QUESTIONS\n",
    "\n",
    "sel_df.loc[\n",
    "    (sel_df[\"is_correct\"] == False) & (sel_df[\"is_near_correct\"] == True),\n",
    "    [\"question\", \"prediction\", \"true_answer\"],\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Average score:'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_correct</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>agent_name</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gpt-4o_03_sept_orchestrator</th>\n",
       "      <td>0.409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar</th>\n",
       "      <td>0.333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar</th>\n",
       "      <td>0.353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>react_code_gpt-4o_03_sept_managedagent-summary_planning</th>\n",
       "      <td>0.433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar</th>\n",
       "      <td>0.366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>react_code_gpt4o_23-june_planning2_newprompt5</th>\n",
       "      <td>0.442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags</th>\n",
       "      <td>0.426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch</th>\n",
       "      <td>0.290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags</th>\n",
       "      <td>0.333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>react_code_gpt4o_6_aug_planning_nogrammar</th>\n",
       "      <td>0.366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>react_code_llama-31-70B_06_sept_managedagent-summary_planning</th>\n",
       "      <td>0.172</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                   is_correct\n",
       "agent_name                                                                                   \n",
       "gpt-4o_03_sept_orchestrator                                                             0.409\n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar                 0.333\n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar                  0.353\n",
       "react_code_gpt-4o_03_sept_managedagent-summary_planning                                 0.433\n",
       "react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar                               0.366\n",
       "react_code_gpt4o_23-june_planning2_newprompt5                                           0.442\n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags                            0.426\n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch       0.290\n",
       "react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags                           0.333\n",
       "react_code_gpt4o_6_aug_planning_nogrammar                                               0.366\n",
       "react_code_llama-31-70B_06_sept_managedagent-summary_planning                           0.172"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>is_correct</th>\n",
       "      <th>is_near_correct</th>\n",
       "      <th>count_steps</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>agent_name</th>\n",
       "      <th>task</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">gpt-4o_03_sept_orchestrator</th>\n",
       "      <th>1</th>\n",
       "      <td>0.528302</td>\n",
       "      <td>0.528302</td>\n",
       "      <td>3.679245</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.435294</td>\n",
       "      <td>0.447059</td>\n",
       "      <td>3.658824</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>3.923077</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar</th>\n",
       "      <th>1</th>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.448980</td>\n",
       "      <td>7.571429</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>9.740741</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.130435</td>\n",
       "      <td>0.130435</td>\n",
       "      <td>12.478261</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar</th>\n",
       "      <th>1</th>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>8.333333</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.344828</td>\n",
       "      <td>0.344828</td>\n",
       "      <td>9.310345</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.857143</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">react_code_gpt-4o_03_sept_managedagent-summary_planning</th>\n",
       "      <th>1</th>\n",
       "      <td>0.566038</td>\n",
       "      <td>0.566038</td>\n",
       "      <td>7.471698</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.423529</td>\n",
       "      <td>0.435294</td>\n",
       "      <td>8.823529</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.192308</td>\n",
       "      <td>0.192308</td>\n",
       "      <td>11.692308</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar</th>\n",
       "      <th>1</th>\n",
       "      <td>0.433962</td>\n",
       "      <td>0.452830</td>\n",
       "      <td>5.660377</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.376471</td>\n",
       "      <td>0.376471</td>\n",
       "      <td>6.717647</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.192308</td>\n",
       "      <td>0.192308</td>\n",
       "      <td>7.038462</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">react_code_gpt4o_23-june_planning2_newprompt5</th>\n",
       "      <th>1</th>\n",
       "      <td>0.584906</td>\n",
       "      <td>0.584906</td>\n",
       "      <td>6.566038</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.430233</td>\n",
       "      <td>0.430233</td>\n",
       "      <td>9.360465</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.192308</td>\n",
       "      <td>0.192308</td>\n",
       "      <td>11.576923</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags</th>\n",
       "      <th>1</th>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>6.727273</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.486486</td>\n",
       "      <td>0.513514</td>\n",
       "      <td>9.054054</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>10.555556</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch</th>\n",
       "      <th>1</th>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.433333</td>\n",
       "      <td>7.566667</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.296296</td>\n",
       "      <td>0.296296</td>\n",
       "      <td>8.296296</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.062500</td>\n",
       "      <td>0.062500</td>\n",
       "      <td>10.625000</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags</th>\n",
       "      <th>1</th>\n",
       "      <td>0.452381</td>\n",
       "      <td>0.452381</td>\n",
       "      <td>7.142857</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.323944</td>\n",
       "      <td>0.352113</td>\n",
       "      <td>8.323944</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>11.181818</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">react_code_gpt4o_6_aug_planning_nogrammar</th>\n",
       "      <th>1</th>\n",
       "      <td>0.433962</td>\n",
       "      <td>0.433962</td>\n",
       "      <td>8.018868</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.388235</td>\n",
       "      <td>0.411765</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.153846</td>\n",
       "      <td>0.153846</td>\n",
       "      <td>13.846154</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">react_code_llama-31-70B_06_sept_managedagent-summary_planning</th>\n",
       "      <th>1</th>\n",
       "      <td>0.055556</td>\n",
       "      <td>0.055556</td>\n",
       "      <td>6.722222</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.242424</td>\n",
       "      <td>0.242424</td>\n",
       "      <td>7.666667</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>10.857143</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                        is_correct  \\\n",
       "agent_name                                                                        task               \n",
       "gpt-4o_03_sept_orchestrator                                                       1       0.528302   \n",
       "                                                                                  2       0.435294   \n",
       "                                                                                  3       0.076923   \n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar           1       0.428571   \n",
       "                                                                                  2       0.333333   \n",
       "                                                                                  3       0.130435   \n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar            1       0.533333   \n",
       "                                                                                  2       0.344828   \n",
       "                                                                                  3       0.000000   \n",
       "react_code_gpt-4o_03_sept_managedagent-summary_planning                           1       0.566038   \n",
       "                                                                                  2       0.423529   \n",
       "                                                                                  3       0.192308   \n",
       "react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar                         1       0.433962   \n",
       "                                                                                  2       0.376471   \n",
       "                                                                                  3       0.192308   \n",
       "react_code_gpt4o_23-june_planning2_newprompt5                                     1       0.584906   \n",
       "                                                                                  2       0.430233   \n",
       "                                                                                  3       0.192308   \n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags                      1       0.454545   \n",
       "                                                                                  2       0.486486   \n",
       "                                                                                  3       0.111111   \n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch 1       0.400000   \n",
       "                                                                                  2       0.296296   \n",
       "                                                                                  3       0.062500   \n",
       "react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags                     1       0.452381   \n",
       "                                                                                  2       0.323944   \n",
       "                                                                                  3       0.136364   \n",
       "react_code_gpt4o_6_aug_planning_nogrammar                                         1       0.433962   \n",
       "                                                                                  2       0.388235   \n",
       "                                                                                  3       0.153846   \n",
       "react_code_llama-31-70B_06_sept_managedagent-summary_planning                     1       0.055556   \n",
       "                                                                                  2       0.242424   \n",
       "                                                                                  3       0.142857   \n",
       "\n",
       "                                                                                        is_near_correct  \\\n",
       "agent_name                                                                        task                    \n",
       "gpt-4o_03_sept_orchestrator                                                       1            0.528302   \n",
       "                                                                                  2            0.447059   \n",
       "                                                                                  3            0.076923   \n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar           1            0.448980   \n",
       "                                                                                  2            0.333333   \n",
       "                                                                                  3            0.130435   \n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar            1            0.533333   \n",
       "                                                                                  2            0.344828   \n",
       "                                                                                  3            0.000000   \n",
       "react_code_gpt-4o_03_sept_managedagent-summary_planning                           1            0.566038   \n",
       "                                                                                  2            0.435294   \n",
       "                                                                                  3            0.192308   \n",
       "react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar                         1            0.452830   \n",
       "                                                                                  2            0.376471   \n",
       "                                                                                  3            0.192308   \n",
       "react_code_gpt4o_23-june_planning2_newprompt5                                     1            0.584906   \n",
       "                                                                                  2            0.430233   \n",
       "                                                                                  3            0.192308   \n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags                      1            0.454545   \n",
       "                                                                                  2            0.513514   \n",
       "                                                                                  3            0.111111   \n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch 1            0.433333   \n",
       "                                                                                  2            0.296296   \n",
       "                                                                                  3            0.062500   \n",
       "react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags                     1            0.452381   \n",
       "                                                                                  2            0.352113   \n",
       "                                                                                  3            0.136364   \n",
       "react_code_gpt4o_6_aug_planning_nogrammar                                         1            0.433962   \n",
       "                                                                                  2            0.411765   \n",
       "                                                                                  3            0.153846   \n",
       "react_code_llama-31-70B_06_sept_managedagent-summary_planning                     1            0.055556   \n",
       "                                                                                  2            0.242424   \n",
       "                                                                                  3            0.142857   \n",
       "\n",
       "                                                                                        count_steps  \\\n",
       "agent_name                                                                        task                \n",
       "gpt-4o_03_sept_orchestrator                                                       1        3.679245   \n",
       "                                                                                  2        3.658824   \n",
       "                                                                                  3        3.923077   \n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar           1        7.571429   \n",
       "                                                                                  2        9.740741   \n",
       "                                                                                  3       12.478261   \n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar            1        8.333333   \n",
       "                                                                                  2        9.310345   \n",
       "                                                                                  3       10.857143   \n",
       "react_code_gpt-4o_03_sept_managedagent-summary_planning                           1        7.471698   \n",
       "                                                                                  2        8.823529   \n",
       "                                                                                  3       11.692308   \n",
       "react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar                         1        5.660377   \n",
       "                                                                                  2        6.717647   \n",
       "                                                                                  3        7.038462   \n",
       "react_code_gpt4o_23-june_planning2_newprompt5                                     1        6.566038   \n",
       "                                                                                  2        9.360465   \n",
       "                                                                                  3       11.576923   \n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags                      1        6.727273   \n",
       "                                                                                  2        9.054054   \n",
       "                                                                                  3       10.555556   \n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch 1        7.566667   \n",
       "                                                                                  2        8.296296   \n",
       "                                                                                  3       10.625000   \n",
       "react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags                     1        7.142857   \n",
       "                                                                                  2        8.323944   \n",
       "                                                                                  3       11.181818   \n",
       "react_code_gpt4o_6_aug_planning_nogrammar                                         1        8.018868   \n",
       "                                                                                  2        9.000000   \n",
       "                                                                                  3       13.846154   \n",
       "react_code_llama-31-70B_06_sept_managedagent-summary_planning                     1        6.722222   \n",
       "                                                                                  2        7.666667   \n",
       "                                                                                  3       10.857143   \n",
       "\n",
       "                                                                                        count  \n",
       "agent_name                                                                        task         \n",
       "gpt-4o_03_sept_orchestrator                                                       1        53  \n",
       "                                                                                  2        85  \n",
       "                                                                                  3        26  \n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar           1        49  \n",
       "                                                                                  2        81  \n",
       "                                                                                  3        23  \n",
       "react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar            1        15  \n",
       "                                                                                  2        29  \n",
       "                                                                                  3         7  \n",
       "react_code_gpt-4o_03_sept_managedagent-summary_planning                           1        53  \n",
       "                                                                                  2        85  \n",
       "                                                                                  3        26  \n",
       "react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar                         1        53  \n",
       "                                                                                  2        85  \n",
       "                                                                                  3        26  \n",
       "react_code_gpt4o_23-june_planning2_newprompt5                                     1        53  \n",
       "                                                                                  2        86  \n",
       "                                                                                  3        26  \n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags                      1        22  \n",
       "                                                                                  2        37  \n",
       "                                                                                  3         9  \n",
       "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch 1        30  \n",
       "                                                                                  2        54  \n",
       "                                                                                  3        16  \n",
       "react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags                     1        42  \n",
       "                                                                                  2        71  \n",
       "                                                                                  3        22  \n",
       "react_code_gpt4o_6_aug_planning_nogrammar                                         1        53  \n",
       "                                                                                  2        85  \n",
       "                                                                                  3        26  \n",
       "react_code_llama-31-70B_06_sept_managedagent-summary_planning                     1        18  \n",
       "                                                                                  2        33  \n",
       "                                                                                  3         7  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(\"Average score:\", sel_df.groupby(\"agent_name\")[[\"is_correct\"]].mean().round(3))\n",
    "display(\n",
    "    sel_df.groupby([\"agent_name\", \"task\"])[\n",
    "        [\"is_correct\", \"is_near_correct\", \"count_steps\", \"question\"]\n",
    "    ]\n",
    "    .agg(\n",
    "        {\n",
    "            \"is_correct\": \"mean\",\n",
    "            \"is_near_correct\": \"mean\",\n",
    "            \"count_steps\": \"mean\",\n",
    "            \"question\": \"count\",\n",
    "        }\n",
    "    )\n",
    "    .rename(columns={\"question\": \"count\"})\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "customdata": [
          [
           "A paper about AI regulation that was originally su"
          ],
          [
           "I’m researching species that became invasive after"
          ],
          [
           "If we assume all articles published by Nature in 2"
          ],
          [
           "In Unlambda, what exact charcter or text needs to "
          ],
          [
           "If Eliud Kipchoge could maintain his record-making"
          ],
          [
           "The attached spreadsheet shows the inventory for a"
          ],
          [
           "How many studio albums were published by Mercedes "
          ],
          [
           "The object in the British Museum's collection with"
          ],
          [
           "According to github, when was Regression added to "
          ],
          [
           "Here's a fun riddle that I think you'll enjoy.\n\nYo"
          ],
          [
           "In July 2, 1959 United States standards for grades"
          ],
          [
           "Using the Biopython library in Python, parse the P"
          ],
          [
           "What are the EC numbers of the two most commonly u"
          ],
          [
           "In April of 1977, who was the Prime Minister of th"
          ],
          [
           "What's the last line of the rhyme under the flavor"
          ],
          [
           "Use density measures from the chemistry materials "
          ],
          [
           "What was the volume in m^3 of the fish bag that wa"
          ],
          [
           "What is the average number of pre-2020 works on th"
          ],
          [
           "In the video https://www.youtube.com/watch?v=L1vXC"
          ],
          [
           "Of the authors (First M. Last) that worked on the "
          ],
          [
           "When you take the average of the standard populati"
          ],
          [
           "Assuming scientists in the famous youtube video Th"
          ],
          [
           "In Series 9, Episode 11 of Doctor Who, the Doctor "
          ],
          [
           "In terms of geographical distance between capital "
          ],
          [
           "In the NCATS PubChem compound database for Food Ad"
          ],
          [
           "I need to fact-check a citation. This is the citat"
          ],
          [
           "Which contributor to the version of OpenCV where s"
          ],
          [
           "What integer-rounded percentage of the total lengt"
          ],
          [
           "An office held a Secret Santa gift exchange where "
          ],
          [
           "What is the maximum length in meters of #9 in the "
          ],
          [
           "What two-word type of model did Manash Pratim Kash"
          ],
          [
           "What animals that were mentioned in both Ilias Lag"
          ],
          [
           "How many High Energy Physics - Lattice articles li"
          ],
          [
           "The photograph in the Whitney Museum of American A"
          ],
          [
           ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht eti"
          ],
          [
           "What is the minimum number of page links a person "
          ],
          [
           "Each cell in the attached spreadsheet represents a"
          ],
          [
           "Which of the text elements under CATEGORIES in the"
          ],
          [
           "I went to Virtue restaurant & bar in Chicago for m"
          ],
          [
           "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) "
          ],
          [
           "My family reunion is this week, and I was assigned"
          ],
          [
           "In Emily Midkiff's June 2014 article in a journal "
          ],
          [
           "It is 1999. Before you party like it is 1999, plea"
          ],
          [
           "Under DDC 633 on Bielefeld University Library's BA"
          ],
          [
           "In the 2018 VSCode blog post on replit.com, what w"
          ],
          [
           "Compute the check digit the Tropicos ID for the Or"
          ],
          [
           "What time was the Tri-Rail train that carried the "
          ],
          [
           "Could you help me out with this assignment? Our pr"
          ],
          [
           "How many applicants for the job in the PDF are onl"
          ],
          [
           "In Valentina Re’s contribution to the 2017 book “W"
          ],
          [
           "In the fictional language of Tizin, basic sentence"
          ],
          [
           "The Metropolitan Museum of Art has a portrait in i"
          ],
          [
           "In Nature journal's Scientific Reports conference "
          ],
          [
           "The attached file contains a list of vendors in th"
          ],
          [
           "According to Google Finance, when was the first ye"
          ],
          [
           "Review the chess position provided in the image. I"
          ],
          [
           "According to Box Office Mojo's 2020 Worldwide Box "
          ],
          [
           "In the year 2022, and before December, what does \""
          ],
          [
           "Who nominated the only Featured Article on English"
          ],
          [
           "What writer is quoted by Merriam-Webster for the W"
          ],
          [
           "How many pages if the 2023 IPCC report (85 pages v"
          ],
          [
           "Given this table defining * on the set S = {a, b, "
          ],
          [
           "The following numbers function similarly to ISBN 1"
          ],
          [
           "How many images are there in the latest 2022 Lego "
          ],
          [
           "The attached file shows a list of books in the col"
          ],
          [
           "I was trying to remember how well the Cheater Beat"
          ],
          [
           "As a comma separated list with no whitespace, usin"
          ],
          [
           "On a leap day before the year 2008, a joke was rem"
          ],
          [
           "What is the volume in milliliters of a system comp"
          ],
          [
           "The Latin root of the Yola word \"gimlie\" shares a "
          ],
          [
           "Find the value of x to the nearest tenth: Lx = (d/"
          ],
          [
           "In the endnote found in the second-to-last paragra"
          ],
          [
           "Using bass clef notes, what is the age of someone "
          ],
          [
           "On July 15, 2008, Phys.org published an article ab"
          ],
          [
           "The attached file lists accommodations in the reso"
          ],
          [
           "In the NIH translation of the original 1913 Michae"
          ],
          [
           "How many edits were made to the Wikipedia page on "
          ],
          [
           "You are a telecommunications engineer who wants to"
          ],
          [
           "If there is anything that doesn't make sense in th"
          ],
          [
           "I was referencing each of the tables in the file f"
          ],
          [
           "How many nonindigenous crocodiles were found in Fl"
          ],
          [
           "The work referenced in footnote 397 of Federico La"
          ],
          [
           "As of the 2020 census, what was the population dif"
          ],
          [
           "How many slides in this PowerPoint presentation me"
          ],
          [
           "What percentage of the total penguin population ac"
          ],
          [
           "Given $x_0 = -5$ and $f(x) = x^3 + 4x^2 - 3x + 8$,"
          ],
          [
           "You are Van Helsing, a renowned vampire hunter. A "
          ],
          [
           "Examine the video at https://www.youtube.com/watch"
          ],
          [
           "The attached file shows the locomotives in the col"
          ],
          [
           "This is a secret message my friend gave me. It say"
          ],
          [
           "What is the area of the green polygon in the attac"
          ],
          [
           "According to wikipedia, how many Asian countries s"
          ],
          [
           "The attached spreadsheet contains the sales of men"
          ],
          [
           "Who composed the song that was performed by a roos"
          ],
          [
           "You are given this Excel file as a map. You start "
          ],
          [
           "I thought we could try a fun word puzzle together "
          ],
          [
           "What is the surname of the equine veterinarian men"
          ],
          [
           "According to the World Bank, which countries had g"
          ],
          [
           "I’m thinking about selling my home, so I want to l"
          ],
          [
           "I'm making a grocery list for my mom, but she's a "
          ],
          [
           "How many times was a Twitter/X post cited as a ref"
          ],
          [
           "On ScienceDirect, what is the difference to 3 deci"
          ],
          [
           "What is the last word before the second chorus of "
          ],
          [
           "Look at the attached image. The quiz is scored as "
          ],
          [
           "Which of the fruits shown in the 2008 painting \"Em"
          ],
          [
           "Hi, I'm making a pie but I could use some help wit"
          ],
          [
           "The attached image contains a Python script. Run t"
          ],
          [
           "I have the Standard plan in the image below, and I"
          ],
          [
           "The attached PDF lists accommodations in the resor"
          ],
          [
           "The year is 2022. I am at the National Air and Spa"
          ],
          [
           "In the Scikit-Learn July 2017 changelog, what othe"
          ],
          [
           "It's May 2023, and I'm about to drive across the U"
          ],
          [
           "Who did the actor who played Ray in the Polish-lan"
          ],
          [
           "What is the latest chronological year date written"
          ],
          [
           "The YouTube channel Game Grumps began a Let’s Play"
          ],
          [
           "This spreadsheet contains a list of clients for a "
          ],
          [
           "On the BBC Earth YouTube video of the Top 5 Sillie"
          ],
          [
           "The book with the doi 10.1353/book.24372 concerns "
          ],
          [
           "The longest-lived vertebrate is named after an isl"
          ],
          [
           "What is the final numeric output from the attached"
          ],
          [
           "On the DeepFruits fruit detection graph on Connect"
          ],
          [
           "During the first week of August 2015, one of the N"
          ],
          [
           "How many more blocks (also denoted as layers) in B"
          ],
          [
           "Bob was invited to participate in a game show, and"
          ],
          [
           "Pull out the sentence in the following 5x7 block o"
          ],
          [
           "All of the individuals who formally held the posit"
          ],
          [
           "The attached spreadsheet contains a list of books "
          ],
          [
           "Consider the following symbols: 𒐜  𒐐𒐚\n\nThis is a n"
          ],
          [
           "On Cornell Law School website's legal information "
          ],
          [
           "According to the USGS, in what year was the Americ"
          ],
          [
           "Of the cities within the United States where U.S. "
          ],
          [
           "Eva Draconis has a personal website which can be a"
          ],
          [
           "The brand that makes these harnesses the dogs are "
          ],
          [
           "According to Girls Who Code, how long did it take "
          ],
          [
           "What was the complete title of the book in which t"
          ],
          [
           "As of August 2023, who is the only winner of the U"
          ],
          [
           "How many at bats did the Yankee with the most walk"
          ],
          [
           "The cover of the August 2021 issue of Vogue shows "
          ],
          [
           "In Audre Lorde’s poem “Father Son and Holy Ghost”,"
          ],
          [
           "I'm curious about how much information is availabl"
          ],
          [
           "The attached spreadsheet lists the locomotives own"
          ],
          [
           "What is the absolute difference in tens of thousan"
          ],
          [
           "The attached file lists the locomotives owned by a"
          ],
          [
           "Hi, I was out sick from my classes on Friday, so I"
          ],
          [
           "When was a picture of St. Thomas Aquinas first add"
          ],
          [
           "A 5-man group made up of one tank, one healer, and"
          ],
          [
           "On June 6, 2023, an article by Carolyn Collins Pet"
          ],
          [
           "According to Openreview.net, at the NeurIPS 2022 C"
          ],
          [
           "Take the gender split from the 2011 Bulgarian cens"
          ],
          [
           "What was the actual enrollment count of the clinic"
          ],
          [
           "I'd like to learn more about some popular reality "
          ],
          [
           "Where were the Vietnamese specimens described by K"
          ],
          [
           "A standard Rubik’s cube has been broken into cubes"
          ],
          [
           "What country had the least number of athletes at t"
          ],
          [
           "I read a paper about multiwavelength observations "
          ],
          [
           "Who are the pitchers with the number before and af"
          ],
          [
           "The attached Excel file contains the sales of menu"
          ],
          [
           "What is the first name of the only Malko Competiti"
          ],
          [
           "In the YouTube 360 VR video from March 2018 narrat"
          ],
          [
           "In NASA's Astronomy Picture of the Day on 2006 Jan"
          ],
          [
           "In the film Goldfinger, what color was the object "
          ],
          [
           "As of May 2023, how many stops are between South S"
          ],
          [
           "In the 2015 Metropolitan Museum of Art exhibition "
          ],
          [
           "At the two-minute mark in the YouTube video upload"
          ]
         ],
         "hovertemplate": "agent_name=gpt-4o_03_sept_orchestrator<br>index=%{x}<br>is_correct=%{y}<br>question=%{customdata[0]}<extra></extra>",
         "legendgroup": "gpt-4o_03_sept_orchestrator",
         "line": {
          "color": "#636efa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "gpt-4o_03_sept_orchestrator",
         "showlegend": true,
         "type": "scattergl",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128,
          129,
          130,
          131,
          132,
          133,
          134,
          135,
          136,
          137,
          138,
          139,
          140,
          141,
          142,
          143,
          144,
          145,
          146,
          147,
          148,
          149,
          150,
          151,
          152,
          153,
          154,
          155,
          156,
          157,
          158,
          159,
          160,
          161,
          162,
          163
         ],
         "xaxis": "x",
         "y": [
          1,
          1,
          1,
          0.75,
          0.6,
          0.6666666666666666,
          0.7142857142857143,
          0.75,
          0.6666666666666666,
          0.6,
          0.5454545454545454,
          0.5,
          0.46153846153846156,
          0.42857142857142855,
          0.4,
          0.4375,
          0.47058823529411764,
          0.4444444444444444,
          0.42105263157894735,
          0.4,
          0.38095238095238093,
          0.36363636363636365,
          0.34782608695652173,
          0.375,
          0.36,
          0.38461538461538464,
          0.4074074074074074,
          0.42857142857142855,
          0.41379310344827586,
          0.4,
          0.41935483870967744,
          0.4375,
          0.42424242424242425,
          0.4411764705882353,
          0.45714285714285713,
          0.4444444444444444,
          0.43243243243243246,
          0.42105263157894735,
          0.41025641025641024,
          0.425,
          0.43902439024390244,
          0.4523809523809524,
          0.4418604651162791,
          0.4318181818181818,
          0.4222222222222222,
          0.43478260869565216,
          0.425531914893617,
          0.4375,
          0.42857142857142855,
          0.44,
          0.43137254901960786,
          0.4423076923076923,
          0.4528301886792453,
          0.46296296296296297,
          0.45454545454545453,
          0.44642857142857145,
          0.43859649122807015,
          0.43103448275862066,
          0.4406779661016949,
          0.45,
          0.45901639344262296,
          0.46774193548387094,
          0.4603174603174603,
          0.453125,
          0.4461538461538462,
          0.4393939393939394,
          0.43283582089552236,
          0.4411764705882353,
          0.43478260869565216,
          0.42857142857142855,
          0.4225352112676056,
          0.4166666666666667,
          0.410958904109589,
          0.4189189189189189,
          0.41333333333333333,
          0.40789473684210525,
          0.4025974025974026,
          0.3974358974358974,
          0.4050632911392405,
          0.4,
          0.3950617283950617,
          0.3902439024390244,
          0.3855421686746988,
          0.39285714285714285,
          0.38823529411764707,
          0.3953488372093023,
          0.39080459770114945,
          0.3977272727272727,
          0.4044943820224719,
          0.4,
          0.3956043956043956,
          0.391304347826087,
          0.3978494623655914,
          0.40425531914893614,
          0.4,
          0.3958333333333333,
          0.3917525773195876,
          0.3877551020408163,
          0.3838383838383838,
          0.38,
          0.37623762376237624,
          0.37254901960784315,
          0.36893203883495146,
          0.36538461538461536,
          0.3619047619047619,
          0.36792452830188677,
          0.37383177570093457,
          0.37962962962962965,
          0.3853211009174312,
          0.38181818181818183,
          0.3783783783783784,
          0.375,
          0.3805309734513274,
          0.37719298245614036,
          0.3739130434782609,
          0.3706896551724138,
          0.37606837606837606,
          0.3813559322033898,
          0.3865546218487395,
          0.39166666666666666,
          0.3884297520661157,
          0.39344262295081966,
          0.3983739837398374,
          0.4032258064516129,
          0.408,
          0.40476190476190477,
          0.4015748031496063,
          0.3984375,
          0.3953488372093023,
          0.4,
          0.3969465648854962,
          0.3939393939393939,
          0.39097744360902253,
          0.3880597014925373,
          0.3851851851851852,
          0.3897058823529412,
          0.39416058394160586,
          0.391304347826087,
          0.39568345323741005,
          0.39285714285714285,
          0.3971631205673759,
          0.4014084507042254,
          0.40559440559440557,
          0.4097222222222222,
          0.4068965517241379,
          0.4041095890410959,
          0.4013605442176871,
          0.39864864864864863,
          0.40268456375838924,
          0.4066666666666667,
          0.40397350993377484,
          0.40789473684210525,
          0.40522875816993464,
          0.4090909090909091,
          0.4064516129032258,
          0.40384615384615385,
          0.40764331210191085,
          0.41139240506329117,
          0.4088050314465409,
          0.40625,
          0.40993788819875776,
          0.41358024691358025,
          0.4110429447852761,
          0.40853658536585363
         ],
         "yaxis": "y"
        },
        {
         "customdata": [
          [
           "A paper about AI regulation that was originally su"
          ],
          [
           "I’m researching species that became invasive after"
          ],
          [
           "If we assume all articles published by Nature in 2"
          ],
          [
           "In Unlambda, what exact charcter or text needs to "
          ],
          [
           "If Eliud Kipchoge could maintain his record-making"
          ],
          [
           "The attached spreadsheet shows the inventory for a"
          ],
          [
           "How many studio albums were published by Mercedes "
          ],
          [
           "The object in the British Museum's collection with"
          ],
          [
           "According to github, when was Regression added to "
          ],
          [
           "Here's a fun riddle that I think you'll enjoy.\n\nYo"
          ],
          [
           "In July 2, 1959 United States standards for grades"
          ],
          [
           "Using the Biopython library in Python, parse the P"
          ],
          [
           "What are the EC numbers of the two most commonly u"
          ],
          [
           "In April of 1977, who was the Prime Minister of th"
          ],
          [
           "What's the last line of the rhyme under the flavor"
          ],
          [
           "Use density measures from the chemistry materials "
          ],
          [
           "What was the volume in m^3 of the fish bag that wa"
          ],
          [
           "What is the average number of pre-2020 works on th"
          ],
          [
           "In the video https://www.youtube.com/watch?v=L1vXC"
          ],
          [
           "Of the authors (First M. Last) that worked on the "
          ],
          [
           "When you take the average of the standard populati"
          ],
          [
           "Assuming scientists in the famous youtube video Th"
          ],
          [
           "In Series 9, Episode 11 of Doctor Who, the Doctor "
          ],
          [
           "In terms of geographical distance between capital "
          ],
          [
           "In the NCATS PubChem compound database for Food Ad"
          ],
          [
           "I need to fact-check a citation. This is the citat"
          ],
          [
           "Which contributor to the version of OpenCV where s"
          ],
          [
           "What integer-rounded percentage of the total lengt"
          ],
          [
           "An office held a Secret Santa gift exchange where "
          ],
          [
           "What is the maximum length in meters of #9 in the "
          ],
          [
           "What two-word type of model did Manash Pratim Kash"
          ],
          [
           "What animals that were mentioned in both Ilias Lag"
          ],
          [
           "How many High Energy Physics - Lattice articles li"
          ],
          [
           "The photograph in the Whitney Museum of American A"
          ],
          [
           ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht eti"
          ],
          [
           "What is the minimum number of page links a person "
          ],
          [
           "Each cell in the attached spreadsheet represents a"
          ],
          [
           "Which of the text elements under CATEGORIES in the"
          ],
          [
           "I went to Virtue restaurant & bar in Chicago for m"
          ],
          [
           "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) "
          ],
          [
           "My family reunion is this week, and I was assigned"
          ],
          [
           "In Emily Midkiff's June 2014 article in a journal "
          ],
          [
           "It is 1999. Before you party like it is 1999, plea"
          ],
          [
           "Under DDC 633 on Bielefeld University Library's BA"
          ],
          [
           "In the 2018 VSCode blog post on replit.com, what w"
          ],
          [
           "Compute the check digit the Tropicos ID for the Or"
          ],
          [
           "What time was the Tri-Rail train that carried the "
          ],
          [
           "Could you help me out with this assignment? Our pr"
          ],
          [
           "How many applicants for the job in the PDF are onl"
          ],
          [
           "In Valentina Re’s contribution to the 2017 book “W"
          ],
          [
           "In the fictional language of Tizin, basic sentence"
          ],
          [
           "The Metropolitan Museum of Art has a portrait in i"
          ],
          [
           "In Nature journal's Scientific Reports conference "
          ],
          [
           "The attached file contains a list of vendors in th"
          ],
          [
           "According to Google Finance, when was the first ye"
          ],
          [
           "Review the chess position provided in the image. I"
          ],
          [
           "According to Box Office Mojo's 2020 Worldwide Box "
          ],
          [
           "In the year 2022, and before December, what does \""
          ],
          [
           "Who nominated the only Featured Article on English"
          ],
          [
           "What writer is quoted by Merriam-Webster for the W"
          ],
          [
           "How many pages if the 2023 IPCC report (85 pages v"
          ],
          [
           "Given this table defining * on the set S = {a, b, "
          ],
          [
           "The following numbers function similarly to ISBN 1"
          ],
          [
           "How many images are there in the latest 2022 Lego "
          ],
          [
           "The attached file shows a list of books in the col"
          ],
          [
           "I was trying to remember how well the Cheater Beat"
          ],
          [
           "As a comma separated list with no whitespace, usin"
          ],
          [
           "On a leap day before the year 2008, a joke was rem"
          ],
          [
           "What is the volume in milliliters of a system comp"
          ],
          [
           "The Latin root of the Yola word \"gimlie\" shares a "
          ],
          [
           "Find the value of x to the nearest tenth: Lx = (d/"
          ],
          [
           "In the endnote found in the second-to-last paragra"
          ],
          [
           "Using bass clef notes, what is the age of someone "
          ],
          [
           "On July 15, 2008, Phys.org published an article ab"
          ],
          [
           "The attached file lists accommodations in the reso"
          ],
          [
           "In the NIH translation of the original 1913 Michae"
          ],
          [
           "How many edits were made to the Wikipedia page on "
          ],
          [
           "You are a telecommunications engineer who wants to"
          ],
          [
           "If there is anything that doesn't make sense in th"
          ],
          [
           "I was referencing each of the tables in the file f"
          ],
          [
           "How many nonindigenous crocodiles were found in Fl"
          ],
          [
           "The work referenced in footnote 397 of Federico La"
          ],
          [
           "As of the 2020 census, what was the population dif"
          ],
          [
           "How many slides in this PowerPoint presentation me"
          ],
          [
           "What percentage of the total penguin population ac"
          ],
          [
           "Given $x_0 = -5$ and $f(x) = x^3 + 4x^2 - 3x + 8$,"
          ],
          [
           "You are Van Helsing, a renowned vampire hunter. A "
          ],
          [
           "Examine the video at https://www.youtube.com/watch"
          ],
          [
           "The attached file shows the locomotives in the col"
          ],
          [
           "This is a secret message my friend gave me. It say"
          ],
          [
           "What is the area of the green polygon in the attac"
          ],
          [
           "According to wikipedia, how many Asian countries s"
          ],
          [
           "The attached spreadsheet contains the sales of men"
          ],
          [
           "Who composed the song that was performed by a roos"
          ],
          [
           "You are given this Excel file as a map. You start "
          ],
          [
           "I thought we could try a fun word puzzle together "
          ],
          [
           "What is the surname of the equine veterinarian men"
          ],
          [
           "According to the World Bank, which countries had g"
          ],
          [
           "I’m thinking about selling my home, so I want to l"
          ],
          [
           "I'm making a grocery list for my mom, but she's a "
          ],
          [
           "How many times was a Twitter/X post cited as a ref"
          ],
          [
           "On ScienceDirect, what is the difference to 3 deci"
          ],
          [
           "What is the last word before the second chorus of "
          ],
          [
           "Look at the attached image. The quiz is scored as "
          ],
          [
           "Which of the fruits shown in the 2008 painting \"Em"
          ],
          [
           "Hi, I'm making a pie but I could use some help wit"
          ],
          [
           "The attached image contains a Python script. Run t"
          ],
          [
           "I have the Standard plan in the image below, and I"
          ],
          [
           "The attached PDF lists accommodations in the resor"
          ],
          [
           "The year is 2022. I am at the National Air and Spa"
          ],
          [
           "In the Scikit-Learn July 2017 changelog, what othe"
          ],
          [
           "It's May 2023, and I'm about to drive across the U"
          ],
          [
           "Who did the actor who played Ray in the Polish-lan"
          ],
          [
           "What is the latest chronological year date written"
          ],
          [
           "The YouTube channel Game Grumps began a Let’s Play"
          ],
          [
           "This spreadsheet contains a list of clients for a "
          ],
          [
           "On the BBC Earth YouTube video of the Top 5 Sillie"
          ],
          [
           "The book with the doi 10.1353/book.24372 concerns "
          ],
          [
           "The longest-lived vertebrate is named after an isl"
          ],
          [
           "What is the final numeric output from the attached"
          ],
          [
           "On the DeepFruits fruit detection graph on Connect"
          ],
          [
           "During the first week of August 2015, one of the N"
          ],
          [
           "How many more blocks (also denoted as layers) in B"
          ],
          [
           "Bob was invited to participate in a game show, and"
          ],
          [
           "Pull out the sentence in the following 5x7 block o"
          ],
          [
           "All of the individuals who formally held the posit"
          ],
          [
           "The attached spreadsheet contains a list of books "
          ],
          [
           "Consider the following symbols: 𒐜  𒐐𒐚\n\nThis is a n"
          ],
          [
           "On Cornell Law School website's legal information "
          ],
          [
           "According to the USGS, in what year was the Americ"
          ],
          [
           "Of the cities within the United States where U.S. "
          ],
          [
           "Eva Draconis has a personal website which can be a"
          ],
          [
           "The brand that makes these harnesses the dogs are "
          ],
          [
           "According to Girls Who Code, how long did it take "
          ],
          [
           "What was the complete title of the book in which t"
          ],
          [
           "As of August 2023, who is the only winner of the U"
          ],
          [
           "How many at bats did the Yankee with the most walk"
          ],
          [
           "The cover of the August 2021 issue of Vogue shows "
          ],
          [
           "In Audre Lorde’s poem “Father Son and Holy Ghost”,"
          ],
          [
           "I'm curious about how much information is availabl"
          ],
          [
           "The attached spreadsheet lists the locomotives own"
          ],
          [
           "What is the absolute difference in tens of thousan"
          ],
          [
           "The attached file lists the locomotives owned by a"
          ],
          [
           "Hi, I was out sick from my classes on Friday, so I"
          ],
          [
           "When was a picture of St. Thomas Aquinas first add"
          ],
          [
           "A 5-man group made up of one tank, one healer, and"
          ],
          [
           "On June 6, 2023, an article by Carolyn Collins Pet"
          ],
          [
           "According to Openreview.net, at the NeurIPS 2022 C"
          ],
          [
           "Take the gender split from the 2011 Bulgarian cens"
          ],
          [
           "What was the actual enrollment count of the clinic"
          ],
          [
           "I'd like to learn more about some popular reality "
          ],
          [
           "Where were the Vietnamese specimens described by K"
          ],
          [
           "A standard Rubik’s cube has been broken into cubes"
          ]
         ],
         "hovertemplate": "agent_name=react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar<br>index=%{x}<br>is_correct=%{y}<br>question=%{customdata[0]}<extra></extra>",
         "legendgroup": "react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar",
         "line": {
          "color": "#EF553B",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "react_code_gpt-4o_02_sept_managedagent-withsummary_noplanning_nogrammar",
         "showlegend": true,
         "type": "scattergl",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128,
          129,
          130,
          131,
          132,
          133,
          134,
          135,
          136,
          137,
          138,
          139,
          140,
          141,
          142,
          143,
          144,
          145,
          146,
          147,
          148,
          149,
          150,
          151,
          152
         ],
         "xaxis": "x",
         "y": [
          0,
          0.5,
          0.3333333333333333,
          0.25,
          0.2,
          0.3333333333333333,
          0.2857142857142857,
          0.375,
          0.3333333333333333,
          0.4,
          0.36363636363636365,
          0.3333333333333333,
          0.3076923076923077,
          0.35714285714285715,
          0.3333333333333333,
          0.375,
          0.4117647058823529,
          0.3888888888888889,
          0.3684210526315789,
          0.35,
          0.3333333333333333,
          0.3181818181818182,
          0.30434782608695654,
          0.2916666666666667,
          0.28,
          0.2692307692307692,
          0.2962962962962963,
          0.32142857142857145,
          0.3103448275862069,
          0.3,
          0.2903225806451613,
          0.28125,
          0.2727272727272727,
          0.2647058823529412,
          0.2857142857142857,
          0.2777777777777778,
          0.2702702702702703,
          0.2631578947368421,
          0.2564102564102564,
          0.25,
          0.2682926829268293,
          0.2619047619047619,
          0.2558139534883721,
          0.25,
          0.26666666666666666,
          0.2826086956521739,
          0.2765957446808511,
          0.2708333333333333,
          0.2653061224489796,
          0.28,
          0.27450980392156865,
          0.28846153846153844,
          0.2830188679245283,
          0.2962962962962963,
          0.2909090909090909,
          0.2857142857142857,
          0.2807017543859649,
          0.27586206896551724,
          0.2711864406779661,
          0.2833333333333333,
          0.2786885245901639,
          0.2903225806451613,
          0.30158730158730157,
          0.296875,
          0.2923076923076923,
          0.2878787878787879,
          0.2835820895522388,
          0.29411764705882354,
          0.2898550724637681,
          0.2857142857142857,
          0.29577464788732394,
          0.2916666666666667,
          0.2876712328767123,
          0.2972972972972973,
          0.29333333333333333,
          0.2894736842105263,
          0.2857142857142857,
          0.2948717948717949,
          0.3037974683544304,
          0.3,
          0.2962962962962963,
          0.2926829268292683,
          0.2891566265060241,
          0.2976190476190476,
          0.29411764705882354,
          0.3023255813953488,
          0.2988505747126437,
          0.3068181818181818,
          0.3146067415730337,
          0.3111111111111111,
          0.3076923076923077,
          0.30434782608695654,
          0.3118279569892473,
          0.3191489361702128,
          0.3157894736842105,
          0.3125,
          0.31958762886597936,
          0.3163265306122449,
          0.31313131313131315,
          0.31,
          0.3069306930693069,
          0.30392156862745096,
          0.30097087378640774,
          0.2980769230769231,
          0.29523809523809524,
          0.3018867924528302,
          0.29906542056074764,
          0.2962962962962963,
          0.30275229357798167,
          0.3090909090909091,
          0.3063063063063063,
          0.30357142857142855,
          0.30973451327433627,
          0.30701754385964913,
          0.30434782608695654,
          0.3017241379310345,
          0.3076923076923077,
          0.3050847457627119,
          0.31092436974789917,
          0.31666666666666665,
          0.3140495867768595,
          0.319672131147541,
          0.3252032520325203,
          0.3225806451612903,
          0.328,
          0.3253968253968254,
          0.3228346456692913,
          0.3203125,
          0.3178294573643411,
          0.3153846153846154,
          0.31297709923664124,
          0.3106060606060606,
          0.3082706766917293,
          0.30597014925373134,
          0.3037037037037037,
          0.3088235294117647,
          0.30656934306569344,
          0.3115942028985507,
          0.30935251798561153,
          0.30714285714285716,
          0.3120567375886525,
          0.31690140845070425,
          0.32167832167832167,
          0.3263888888888889,
          0.32413793103448274,
          0.3219178082191781,
          0.32653061224489793,
          0.32432432432432434,
          0.3288590604026846,
          0.3333333333333333,
          0.33112582781456956,
          0.3355263157894737,
          0.3333333333333333
         ],
         "yaxis": "y"
        },
        {
         "customdata": [
          [
           "A paper about AI regulation that was originally su"
          ],
          [
           "I’m researching species that became invasive after"
          ],
          [
           "If we assume all articles published by Nature in 2"
          ],
          [
           "In Unlambda, what exact charcter or text needs to "
          ],
          [
           "If Eliud Kipchoge could maintain his record-making"
          ],
          [
           "The attached spreadsheet shows the inventory for a"
          ],
          [
           "How many studio albums were published by Mercedes "
          ],
          [
           "The object in the British Museum's collection with"
          ],
          [
           "According to github, when was Regression added to "
          ],
          [
           "Here's a fun riddle that I think you'll enjoy.\n\nYo"
          ],
          [
           "In July 2, 1959 United States standards for grades"
          ],
          [
           "Using the Biopython library in Python, parse the P"
          ],
          [
           "What are the EC numbers of the two most commonly u"
          ],
          [
           "In April of 1977, who was the Prime Minister of th"
          ],
          [
           "What's the last line of the rhyme under the flavor"
          ],
          [
           "Use density measures from the chemistry materials "
          ],
          [
           "What was the volume in m^3 of the fish bag that wa"
          ],
          [
           "What is the average number of pre-2020 works on th"
          ],
          [
           "In the video https://www.youtube.com/watch?v=L1vXC"
          ],
          [
           "Of the authors (First M. Last) that worked on the "
          ],
          [
           "When you take the average of the standard populati"
          ],
          [
           "Assuming scientists in the famous youtube video Th"
          ],
          [
           "In Series 9, Episode 11 of Doctor Who, the Doctor "
          ],
          [
           "In terms of geographical distance between capital "
          ],
          [
           "In the NCATS PubChem compound database for Food Ad"
          ],
          [
           "I need to fact-check a citation. This is the citat"
          ],
          [
           "Which contributor to the version of OpenCV where s"
          ],
          [
           "What integer-rounded percentage of the total lengt"
          ],
          [
           "An office held a Secret Santa gift exchange where "
          ],
          [
           "What is the maximum length in meters of #9 in the "
          ],
          [
           "What two-word type of model did Manash Pratim Kash"
          ],
          [
           "What animals that were mentioned in both Ilias Lag"
          ],
          [
           "How many High Energy Physics - Lattice articles li"
          ],
          [
           "The photograph in the Whitney Museum of American A"
          ],
          [
           ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht eti"
          ],
          [
           "What is the minimum number of page links a person "
          ],
          [
           "Each cell in the attached spreadsheet represents a"
          ],
          [
           "Which of the text elements under CATEGORIES in the"
          ],
          [
           "I went to Virtue restaurant & bar in Chicago for m"
          ],
          [
           "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) "
          ],
          [
           "My family reunion is this week, and I was assigned"
          ],
          [
           "In Emily Midkiff's June 2014 article in a journal "
          ],
          [
           "It is 1999. Before you party like it is 1999, plea"
          ],
          [
           "Under DDC 633 on Bielefeld University Library's BA"
          ],
          [
           "In the 2018 VSCode blog post on replit.com, what w"
          ],
          [
           "Compute the check digit the Tropicos ID for the Or"
          ],
          [
           "What time was the Tri-Rail train that carried the "
          ],
          [
           "Could you help me out with this assignment? Our pr"
          ],
          [
           "How many applicants for the job in the PDF are onl"
          ],
          [
           "In Valentina Re’s contribution to the 2017 book “W"
          ],
          [
           "In the fictional language of Tizin, basic sentence"
          ]
         ],
         "hovertemplate": "agent_name=react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar<br>index=%{x}<br>is_correct=%{y}<br>question=%{customdata[0]}<extra></extra>",
         "legendgroup": "react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar",
         "line": {
          "color": "#00cc96",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "react_code_gpt-4o_02_sept_managedagent-withsummary_planning3_nogrammar",
         "showlegend": true,
         "type": "scattergl",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50
         ],
         "xaxis": "x",
         "y": [
          1,
          0.5,
          0.6666666666666666,
          0.5,
          0.6,
          0.6666666666666666,
          0.7142857142857143,
          0.75,
          0.6666666666666666,
          0.6,
          0.5454545454545454,
          0.5,
          0.46153846153846156,
          0.42857142857142855,
          0.4,
          0.4375,
          0.47058823529411764,
          0.4444444444444444,
          0.42105263157894735,
          0.4,
          0.38095238095238093,
          0.36363636363636365,
          0.34782608695652173,
          0.375,
          0.36,
          0.38461538461538464,
          0.37037037037037035,
          0.35714285714285715,
          0.3448275862068966,
          0.3333333333333333,
          0.3225806451612903,
          0.3125,
          0.30303030303030304,
          0.29411764705882354,
          0.3142857142857143,
          0.3055555555555556,
          0.32432432432432434,
          0.3157894736842105,
          0.3333333333333333,
          0.35,
          0.36585365853658536,
          0.38095238095238093,
          0.37209302325581395,
          0.36363636363636365,
          0.35555555555555557,
          0.3695652173913043,
          0.3617021276595745,
          0.3541666666666667,
          0.3469387755102041,
          0.36,
          0.35294117647058826
         ],
         "yaxis": "y"
        },
        {
         "customdata": [
          [
           "A paper about AI regulation that was originally su"
          ],
          [
           "I’m researching species that became invasive after"
          ],
          [
           "If we assume all articles published by Nature in 2"
          ],
          [
           "In Unlambda, what exact charcter or text needs to "
          ],
          [
           "If Eliud Kipchoge could maintain his record-making"
          ],
          [
           "The attached spreadsheet shows the inventory for a"
          ],
          [
           "How many studio albums were published by Mercedes "
          ],
          [
           "The object in the British Museum's collection with"
          ],
          [
           "According to github, when was Regression added to "
          ],
          [
           "Here's a fun riddle that I think you'll enjoy.\n\nYo"
          ],
          [
           "In July 2, 1959 United States standards for grades"
          ],
          [
           "Using the Biopython library in Python, parse the P"
          ],
          [
           "What are the EC numbers of the two most commonly u"
          ],
          [
           "In April of 1977, who was the Prime Minister of th"
          ],
          [
           "What's the last line of the rhyme under the flavor"
          ],
          [
           "Use density measures from the chemistry materials "
          ],
          [
           "What was the volume in m^3 of the fish bag that wa"
          ],
          [
           "What is the average number of pre-2020 works on th"
          ],
          [
           "In the video https://www.youtube.com/watch?v=L1vXC"
          ],
          [
           "Of the authors (First M. Last) that worked on the "
          ],
          [
           "When you take the average of the standard populati"
          ],
          [
           "Assuming scientists in the famous youtube video Th"
          ],
          [
           "In Series 9, Episode 11 of Doctor Who, the Doctor "
          ],
          [
           "In terms of geographical distance between capital "
          ],
          [
           "In the NCATS PubChem compound database for Food Ad"
          ],
          [
           "I need to fact-check a citation. This is the citat"
          ],
          [
           "Which contributor to the version of OpenCV where s"
          ],
          [
           "What integer-rounded percentage of the total lengt"
          ],
          [
           "An office held a Secret Santa gift exchange where "
          ],
          [
           "What is the maximum length in meters of #9 in the "
          ],
          [
           "What two-word type of model did Manash Pratim Kash"
          ],
          [
           "What animals that were mentioned in both Ilias Lag"
          ],
          [
           "How many High Energy Physics - Lattice articles li"
          ],
          [
           "The photograph in the Whitney Museum of American A"
          ],
          [
           ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht eti"
          ],
          [
           "What is the minimum number of page links a person "
          ],
          [
           "Each cell in the attached spreadsheet represents a"
          ],
          [
           "Which of the text elements under CATEGORIES in the"
          ],
          [
           "I went to Virtue restaurant & bar in Chicago for m"
          ],
          [
           "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) "
          ],
          [
           "My family reunion is this week, and I was assigned"
          ],
          [
           "In Emily Midkiff's June 2014 article in a journal "
          ],
          [
           "It is 1999. Before you party like it is 1999, plea"
          ],
          [
           "Under DDC 633 on Bielefeld University Library's BA"
          ],
          [
           "In the 2018 VSCode blog post on replit.com, what w"
          ],
          [
           "Compute the check digit the Tropicos ID for the Or"
          ],
          [
           "What time was the Tri-Rail train that carried the "
          ],
          [
           "Could you help me out with this assignment? Our pr"
          ],
          [
           "How many applicants for the job in the PDF are onl"
          ],
          [
           "In Valentina Re’s contribution to the 2017 book “W"
          ],
          [
           "In the fictional language of Tizin, basic sentence"
          ],
          [
           "The Metropolitan Museum of Art has a portrait in i"
          ],
          [
           "In Nature journal's Scientific Reports conference "
          ],
          [
           "The attached file contains a list of vendors in th"
          ],
          [
           "According to Google Finance, when was the first ye"
          ],
          [
           "Review the chess position provided in the image. I"
          ],
          [
           "According to Box Office Mojo's 2020 Worldwide Box "
          ],
          [
           "In the year 2022, and before December, what does \""
          ],
          [
           "Who nominated the only Featured Article on English"
          ],
          [
           "What writer is quoted by Merriam-Webster for the W"
          ],
          [
           "How many pages if the 2023 IPCC report (85 pages v"
          ],
          [
           "Given this table defining * on the set S = {a, b, "
          ],
          [
           "The following numbers function similarly to ISBN 1"
          ],
          [
           "How many images are there in the latest 2022 Lego "
          ],
          [
           "The attached file shows a list of books in the col"
          ],
          [
           "I was trying to remember how well the Cheater Beat"
          ],
          [
           "As a comma separated list with no whitespace, usin"
          ],
          [
           "On a leap day before the year 2008, a joke was rem"
          ],
          [
           "What is the volume in milliliters of a system comp"
          ],
          [
           "The Latin root of the Yola word \"gimlie\" shares a "
          ],
          [
           "Find the value of x to the nearest tenth: Lx = (d/"
          ],
          [
           "In the endnote found in the second-to-last paragra"
          ],
          [
           "Using bass clef notes, what is the age of someone "
          ],
          [
           "On July 15, 2008, Phys.org published an article ab"
          ],
          [
           "The attached file lists accommodations in the reso"
          ],
          [
           "In the NIH translation of the original 1913 Michae"
          ],
          [
           "How many edits were made to the Wikipedia page on "
          ],
          [
           "You are a telecommunications engineer who wants to"
          ],
          [
           "If there is anything that doesn't make sense in th"
          ],
          [
           "I was referencing each of the tables in the file f"
          ],
          [
           "How many nonindigenous crocodiles were found in Fl"
          ],
          [
           "The work referenced in footnote 397 of Federico La"
          ],
          [
           "As of the 2020 census, what was the population dif"
          ],
          [
           "How many slides in this PowerPoint presentation me"
          ],
          [
           "What percentage of the total penguin population ac"
          ],
          [
           "Given $x_0 = -5$ and $f(x) = x^3 + 4x^2 - 3x + 8$,"
          ],
          [
           "You are Van Helsing, a renowned vampire hunter. A "
          ],
          [
           "Examine the video at https://www.youtube.com/watch"
          ],
          [
           "The attached file shows the locomotives in the col"
          ],
          [
           "This is a secret message my friend gave me. It say"
          ],
          [
           "What is the area of the green polygon in the attac"
          ],
          [
           "According to wikipedia, how many Asian countries s"
          ],
          [
           "The attached spreadsheet contains the sales of men"
          ],
          [
           "Who composed the song that was performed by a roos"
          ],
          [
           "You are given this Excel file as a map. You start "
          ],
          [
           "I thought we could try a fun word puzzle together "
          ],
          [
           "What is the surname of the equine veterinarian men"
          ],
          [
           "According to the World Bank, which countries had g"
          ],
          [
           "I’m thinking about selling my home, so I want to l"
          ],
          [
           "I'm making a grocery list for my mom, but she's a "
          ],
          [
           "How many times was a Twitter/X post cited as a ref"
          ],
          [
           "On ScienceDirect, what is the difference to 3 deci"
          ],
          [
           "What is the last word before the second chorus of "
          ],
          [
           "Look at the attached image. The quiz is scored as "
          ],
          [
           "Which of the fruits shown in the 2008 painting \"Em"
          ],
          [
           "Hi, I'm making a pie but I could use some help wit"
          ],
          [
           "The attached image contains a Python script. Run t"
          ],
          [
           "I have the Standard plan in the image below, and I"
          ],
          [
           "The attached PDF lists accommodations in the resor"
          ],
          [
           "The year is 2022. I am at the National Air and Spa"
          ],
          [
           "In the Scikit-Learn July 2017 changelog, what othe"
          ],
          [
           "It's May 2023, and I'm about to drive across the U"
          ],
          [
           "Who did the actor who played Ray in the Polish-lan"
          ],
          [
           "What is the latest chronological year date written"
          ],
          [
           "The YouTube channel Game Grumps began a Let’s Play"
          ],
          [
           "This spreadsheet contains a list of clients for a "
          ],
          [
           "On the BBC Earth YouTube video of the Top 5 Sillie"
          ],
          [
           "The book with the doi 10.1353/book.24372 concerns "
          ],
          [
           "The longest-lived vertebrate is named after an isl"
          ],
          [
           "What is the final numeric output from the attached"
          ],
          [
           "On the DeepFruits fruit detection graph on Connect"
          ],
          [
           "During the first week of August 2015, one of the N"
          ],
          [
           "How many more blocks (also denoted as layers) in B"
          ],
          [
           "Bob was invited to participate in a game show, and"
          ],
          [
           "Pull out the sentence in the following 5x7 block o"
          ],
          [
           "All of the individuals who formally held the posit"
          ],
          [
           "The attached spreadsheet contains a list of books "
          ],
          [
           "Consider the following symbols: 𒐜  𒐐𒐚\n\nThis is a n"
          ],
          [
           "On Cornell Law School website's legal information "
          ],
          [
           "According to the USGS, in what year was the Americ"
          ],
          [
           "Of the cities within the United States where U.S. "
          ],
          [
           "Eva Draconis has a personal website which can be a"
          ],
          [
           "The brand that makes these harnesses the dogs are "
          ],
          [
           "According to Girls Who Code, how long did it take "
          ],
          [
           "What was the complete title of the book in which t"
          ],
          [
           "As of August 2023, who is the only winner of the U"
          ],
          [
           "How many at bats did the Yankee with the most walk"
          ],
          [
           "The cover of the August 2021 issue of Vogue shows "
          ],
          [
           "In Audre Lorde’s poem “Father Son and Holy Ghost”,"
          ],
          [
           "I'm curious about how much information is availabl"
          ],
          [
           "The attached spreadsheet lists the locomotives own"
          ],
          [
           "What is the absolute difference in tens of thousan"
          ],
          [
           "The attached file lists the locomotives owned by a"
          ],
          [
           "Hi, I was out sick from my classes on Friday, so I"
          ],
          [
           "When was a picture of St. Thomas Aquinas first add"
          ],
          [
           "A 5-man group made up of one tank, one healer, and"
          ],
          [
           "On June 6, 2023, an article by Carolyn Collins Pet"
          ],
          [
           "According to Openreview.net, at the NeurIPS 2022 C"
          ],
          [
           "Take the gender split from the 2011 Bulgarian cens"
          ],
          [
           "What was the actual enrollment count of the clinic"
          ],
          [
           "I'd like to learn more about some popular reality "
          ],
          [
           "Where were the Vietnamese specimens described by K"
          ],
          [
           "A standard Rubik’s cube has been broken into cubes"
          ],
          [
           "What country had the least number of athletes at t"
          ],
          [
           "I read a paper about multiwavelength observations "
          ],
          [
           "Who are the pitchers with the number before and af"
          ],
          [
           "The attached Excel file contains the sales of menu"
          ],
          [
           "What is the first name of the only Malko Competiti"
          ],
          [
           "In the YouTube 360 VR video from March 2018 narrat"
          ],
          [
           "In NASA's Astronomy Picture of the Day on 2006 Jan"
          ],
          [
           "In the film Goldfinger, what color was the object "
          ],
          [
           "As of May 2023, how many stops are between South S"
          ],
          [
           "In the 2015 Metropolitan Museum of Art exhibition "
          ],
          [
           "At the two-minute mark in the YouTube video upload"
          ]
         ],
         "hovertemplate": "agent_name=react_code_gpt-4o_03_sept_managedagent-summary_planning<br>index=%{x}<br>is_correct=%{y}<br>question=%{customdata[0]}<extra></extra>",
         "legendgroup": "react_code_gpt-4o_03_sept_managedagent-summary_planning",
         "line": {
          "color": "#ab63fa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "react_code_gpt-4o_03_sept_managedagent-summary_planning",
         "showlegend": true,
         "type": "scattergl",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128,
          129,
          130,
          131,
          132,
          133,
          134,
          135,
          136,
          137,
          138,
          139,
          140,
          141,
          142,
          143,
          144,
          145,
          146,
          147,
          148,
          149,
          150,
          151,
          152,
          153,
          154,
          155,
          156,
          157,
          158,
          159,
          160,
          161,
          162,
          163
         ],
         "xaxis": "x",
         "y": [
          1,
          1,
          1,
          0.75,
          0.6,
          0.6666666666666666,
          0.7142857142857143,
          0.75,
          0.6666666666666666,
          0.6,
          0.5454545454545454,
          0.5,
          0.46153846153846156,
          0.42857142857142855,
          0.4,
          0.4375,
          0.47058823529411764,
          0.4444444444444444,
          0.47368421052631576,
          0.5,
          0.47619047619047616,
          0.5,
          0.4782608695652174,
          0.4583333333333333,
          0.44,
          0.46153846153846156,
          0.48148148148148145,
          0.4642857142857143,
          0.4482758620689655,
          0.43333333333333335,
          0.45161290322580644,
          0.4375,
          0.42424242424242425,
          0.4117647058823529,
          0.42857142857142855,
          0.4166666666666667,
          0.43243243243243246,
          0.42105263157894735,
          0.41025641025641024,
          0.425,
          0.43902439024390244,
          0.4523809523809524,
          0.4418604651162791,
          0.4318181818181818,
          0.4222222222222222,
          0.43478260869565216,
          0.425531914893617,
          0.4166666666666667,
          0.40816326530612246,
          0.42,
          0.4117647058823529,
          0.4230769230769231,
          0.4339622641509434,
          0.4444444444444444,
          0.43636363636363634,
          0.42857142857142855,
          0.42105263157894735,
          0.41379310344827586,
          0.423728813559322,
          0.43333333333333335,
          0.4426229508196721,
          0.45161290322580644,
          0.4444444444444444,
          0.4375,
          0.4307692307692308,
          0.42424242424242425,
          0.417910447761194,
          0.4264705882352941,
          0.42028985507246375,
          0.4142857142857143,
          0.4225352112676056,
          0.4166666666666667,
          0.410958904109589,
          0.4189189189189189,
          0.41333333333333333,
          0.40789473684210525,
          0.4025974025974026,
          0.3974358974358974,
          0.4050632911392405,
          0.4,
          0.3950617283950617,
          0.3902439024390244,
          0.3855421686746988,
          0.39285714285714285,
          0.38823529411764707,
          0.3953488372093023,
          0.39080459770114945,
          0.3977272727272727,
          0.4044943820224719,
          0.4,
          0.3956043956043956,
          0.391304347826087,
          0.3978494623655914,
          0.40425531914893614,
          0.4,
          0.3958333333333333,
          0.3917525773195876,
          0.3877551020408163,
          0.3838383838383838,
          0.39,
          0.38613861386138615,
          0.38235294117647056,
          0.3786407766990291,
          0.375,
          0.37142857142857144,
          0.37735849056603776,
          0.38317757009345793,
          0.3888888888888889,
          0.3944954128440367,
          0.4,
          0.3963963963963964,
          0.4017857142857143,
          0.40707964601769914,
          0.40350877192982454,
          0.4,
          0.39655172413793105,
          0.4017094017094017,
          0.4067796610169492,
          0.4117647058823529,
          0.4166666666666667,
          0.4132231404958678,
          0.4180327868852459,
          0.42276422764227645,
          0.41935483870967744,
          0.424,
          0.42063492063492064,
          0.4251968503937008,
          0.4296875,
          0.4263565891472868,
          0.4307692307692308,
          0.42748091603053434,
          0.4318181818181818,
          0.42857142857142855,
          0.4253731343283582,
          0.4222222222222222,
          0.4264705882352941,
          0.4233576642335766,
          0.42028985507246375,
          0.4172661870503597,
          0.4142857142857143,
          0.41843971631205673,
          0.4225352112676056,
          0.42657342657342656,
          0.4305555555555556,
          0.42758620689655175,
          0.4246575342465753,
          0.42857142857142855,
          0.42567567567567566,
          0.42953020134228187,
          0.4266666666666667,
          0.423841059602649,
          0.4276315789473684,
          0.43137254901960786,
          0.43506493506493504,
          0.432258064516129,
          0.42948717948717946,
          0.43312101910828027,
          0.43670886075949367,
          0.44025157232704404,
          0.4375,
          0.43478260869565216,
          0.4382716049382716,
          0.43558282208588955,
          0.4329268292682927
         ],
         "yaxis": "y"
        },
        {
         "customdata": [
          [
           "A paper about AI regulation that was originally su"
          ],
          [
           "I’m researching species that became invasive after"
          ],
          [
           "If we assume all articles published by Nature in 2"
          ],
          [
           "In Unlambda, what exact charcter or text needs to "
          ],
          [
           "If Eliud Kipchoge could maintain his record-making"
          ],
          [
           "The attached spreadsheet shows the inventory for a"
          ],
          [
           "How many studio albums were published by Mercedes "
          ],
          [
           "The object in the British Museum's collection with"
          ],
          [
           "According to github, when was Regression added to "
          ],
          [
           "Here's a fun riddle that I think you'll enjoy.\n\nYo"
          ],
          [
           "In July 2, 1959 United States standards for grades"
          ],
          [
           "Using the Biopython library in Python, parse the P"
          ],
          [
           "What are the EC numbers of the two most commonly u"
          ],
          [
           "In April of 1977, who was the Prime Minister of th"
          ],
          [
           "What's the last line of the rhyme under the flavor"
          ],
          [
           "Use density measures from the chemistry materials "
          ],
          [
           "What was the volume in m^3 of the fish bag that wa"
          ],
          [
           "What is the average number of pre-2020 works on th"
          ],
          [
           "In the video https://www.youtube.com/watch?v=L1vXC"
          ],
          [
           "Of the authors (First M. Last) that worked on the "
          ],
          [
           "When you take the average of the standard populati"
          ],
          [
           "Assuming scientists in the famous youtube video Th"
          ],
          [
           "In Series 9, Episode 11 of Doctor Who, the Doctor "
          ],
          [
           "In terms of geographical distance between capital "
          ],
          [
           "In the NCATS PubChem compound database for Food Ad"
          ],
          [
           "I need to fact-check a citation. This is the citat"
          ],
          [
           "Which contributor to the version of OpenCV where s"
          ],
          [
           "What integer-rounded percentage of the total lengt"
          ],
          [
           "An office held a Secret Santa gift exchange where "
          ],
          [
           "What is the maximum length in meters of #9 in the "
          ],
          [
           "What two-word type of model did Manash Pratim Kash"
          ],
          [
           "What animals that were mentioned in both Ilias Lag"
          ],
          [
           "How many High Energy Physics - Lattice articles li"
          ],
          [
           "The photograph in the Whitney Museum of American A"
          ],
          [
           ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht eti"
          ],
          [
           "What is the minimum number of page links a person "
          ],
          [
           "Each cell in the attached spreadsheet represents a"
          ],
          [
           "Which of the text elements under CATEGORIES in the"
          ],
          [
           "I went to Virtue restaurant & bar in Chicago for m"
          ],
          [
           "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) "
          ],
          [
           "My family reunion is this week, and I was assigned"
          ],
          [
           "In Emily Midkiff's June 2014 article in a journal "
          ],
          [
           "It is 1999. Before you party like it is 1999, plea"
          ],
          [
           "Under DDC 633 on Bielefeld University Library's BA"
          ],
          [
           "In the 2018 VSCode blog post on replit.com, what w"
          ],
          [
           "Compute the check digit the Tropicos ID for the Or"
          ],
          [
           "What time was the Tri-Rail train that carried the "
          ],
          [
           "Could you help me out with this assignment? Our pr"
          ],
          [
           "How many applicants for the job in the PDF are onl"
          ],
          [
           "In Valentina Re’s contribution to the 2017 book “W"
          ],
          [
           "In the fictional language of Tizin, basic sentence"
          ],
          [
           "The Metropolitan Museum of Art has a portrait in i"
          ],
          [
           "In Nature journal's Scientific Reports conference "
          ],
          [
           "The attached file contains a list of vendors in th"
          ],
          [
           "According to Google Finance, when was the first ye"
          ],
          [
           "Review the chess position provided in the image. I"
          ],
          [
           "According to Box Office Mojo's 2020 Worldwide Box "
          ],
          [
           "In the year 2022, and before December, what does \""
          ],
          [
           "Who nominated the only Featured Article on English"
          ],
          [
           "What writer is quoted by Merriam-Webster for the W"
          ],
          [
           "How many pages if the 2023 IPCC report (85 pages v"
          ],
          [
           "Given this table defining * on the set S = {a, b, "
          ],
          [
           "The following numbers function similarly to ISBN 1"
          ],
          [
           "How many images are there in the latest 2022 Lego "
          ],
          [
           "The attached file shows a list of books in the col"
          ],
          [
           "I was trying to remember how well the Cheater Beat"
          ],
          [
           "As a comma separated list with no whitespace, usin"
          ],
          [
           "On a leap day before the year 2008, a joke was rem"
          ],
          [
           "What is the volume in milliliters of a system comp"
          ],
          [
           "The Latin root of the Yola word \"gimlie\" shares a "
          ],
          [
           "Find the value of x to the nearest tenth: Lx = (d/"
          ],
          [
           "In the endnote found in the second-to-last paragra"
          ],
          [
           "Using bass clef notes, what is the age of someone "
          ],
          [
           "On July 15, 2008, Phys.org published an article ab"
          ],
          [
           "The attached file lists accommodations in the reso"
          ],
          [
           "In the NIH translation of the original 1913 Michae"
          ],
          [
           "How many edits were made to the Wikipedia page on "
          ],
          [
           "You are a telecommunications engineer who wants to"
          ],
          [
           "If there is anything that doesn't make sense in th"
          ],
          [
           "I was referencing each of the tables in the file f"
          ],
          [
           "How many nonindigenous crocodiles were found in Fl"
          ],
          [
           "The work referenced in footnote 397 of Federico La"
          ],
          [
           "As of the 2020 census, what was the population dif"
          ],
          [
           "How many slides in this PowerPoint presentation me"
          ],
          [
           "What percentage of the total penguin population ac"
          ],
          [
           "Given $x_0 = -5$ and $f(x) = x^3 + 4x^2 - 3x + 8$,"
          ],
          [
           "You are Van Helsing, a renowned vampire hunter. A "
          ],
          [
           "Examine the video at https://www.youtube.com/watch"
          ],
          [
           "The attached file shows the locomotives in the col"
          ],
          [
           "This is a secret message my friend gave me. It say"
          ],
          [
           "What is the area of the green polygon in the attac"
          ],
          [
           "According to wikipedia, how many Asian countries s"
          ],
          [
           "The attached spreadsheet contains the sales of men"
          ],
          [
           "Who composed the song that was performed by a roos"
          ],
          [
           "You are given this Excel file as a map. You start "
          ],
          [
           "I thought we could try a fun word puzzle together "
          ],
          [
           "What is the surname of the equine veterinarian men"
          ],
          [
           "According to the World Bank, which countries had g"
          ],
          [
           "I’m thinking about selling my home, so I want to l"
          ],
          [
           "I'm making a grocery list for my mom, but she's a "
          ],
          [
           "How many times was a Twitter/X post cited as a ref"
          ],
          [
           "On ScienceDirect, what is the difference to 3 deci"
          ],
          [
           "What is the last word before the second chorus of "
          ],
          [
           "Look at the attached image. The quiz is scored as "
          ],
          [
           "Which of the fruits shown in the 2008 painting \"Em"
          ],
          [
           "Hi, I'm making a pie but I could use some help wit"
          ],
          [
           "The attached image contains a Python script. Run t"
          ],
          [
           "I have the Standard plan in the image below, and I"
          ],
          [
           "The attached PDF lists accommodations in the resor"
          ],
          [
           "The year is 2022. I am at the National Air and Spa"
          ],
          [
           "In the Scikit-Learn July 2017 changelog, what othe"
          ],
          [
           "It's May 2023, and I'm about to drive across the U"
          ],
          [
           "Who did the actor who played Ray in the Polish-lan"
          ],
          [
           "What is the latest chronological year date written"
          ],
          [
           "The YouTube channel Game Grumps began a Let’s Play"
          ],
          [
           "This spreadsheet contains a list of clients for a "
          ],
          [
           "On the BBC Earth YouTube video of the Top 5 Sillie"
          ],
          [
           "The book with the doi 10.1353/book.24372 concerns "
          ],
          [
           "The longest-lived vertebrate is named after an isl"
          ],
          [
           "What is the final numeric output from the attached"
          ],
          [
           "On the DeepFruits fruit detection graph on Connect"
          ],
          [
           "During the first week of August 2015, one of the N"
          ],
          [
           "How many more blocks (also denoted as layers) in B"
          ],
          [
           "Bob was invited to participate in a game show, and"
          ],
          [
           "Pull out the sentence in the following 5x7 block o"
          ],
          [
           "All of the individuals who formally held the posit"
          ],
          [
           "The attached spreadsheet contains a list of books "
          ],
          [
           "Consider the following symbols: 𒐜  𒐐𒐚\n\nThis is a n"
          ],
          [
           "On Cornell Law School website's legal information "
          ],
          [
           "According to the USGS, in what year was the Americ"
          ],
          [
           "Of the cities within the United States where U.S. "
          ],
          [
           "Eva Draconis has a personal website which can be a"
          ],
          [
           "The brand that makes these harnesses the dogs are "
          ],
          [
           "According to Girls Who Code, how long did it take "
          ],
          [
           "What was the complete title of the book in which t"
          ],
          [
           "As of August 2023, who is the only winner of the U"
          ],
          [
           "How many at bats did the Yankee with the most walk"
          ],
          [
           "The cover of the August 2021 issue of Vogue shows "
          ],
          [
           "In Audre Lorde’s poem “Father Son and Holy Ghost”,"
          ],
          [
           "I'm curious about how much information is availabl"
          ],
          [
           "The attached spreadsheet lists the locomotives own"
          ],
          [
           "What is the absolute difference in tens of thousan"
          ],
          [
           "The attached file lists the locomotives owned by a"
          ],
          [
           "Hi, I was out sick from my classes on Friday, so I"
          ],
          [
           "When was a picture of St. Thomas Aquinas first add"
          ],
          [
           "A 5-man group made up of one tank, one healer, and"
          ],
          [
           "On June 6, 2023, an article by Carolyn Collins Pet"
          ],
          [
           "According to Openreview.net, at the NeurIPS 2022 C"
          ],
          [
           "Take the gender split from the 2011 Bulgarian cens"
          ],
          [
           "What was the actual enrollment count of the clinic"
          ],
          [
           "I'd like to learn more about some popular reality "
          ],
          [
           "Where were the Vietnamese specimens described by K"
          ],
          [
           "A standard Rubik’s cube has been broken into cubes"
          ],
          [
           "What country had the least number of athletes at t"
          ],
          [
           "I read a paper about multiwavelength observations "
          ],
          [
           "Who are the pitchers with the number before and af"
          ],
          [
           "The attached Excel file contains the sales of menu"
          ],
          [
           "What is the first name of the only Malko Competiti"
          ],
          [
           "In the YouTube 360 VR video from March 2018 narrat"
          ],
          [
           "In NASA's Astronomy Picture of the Day on 2006 Jan"
          ],
          [
           "In the film Goldfinger, what color was the object "
          ],
          [
           "As of May 2023, how many stops are between South S"
          ],
          [
           "In the 2015 Metropolitan Museum of Art exhibition "
          ],
          [
           "At the two-minute mark in the YouTube video upload"
          ]
         ],
         "hovertemplate": "agent_name=react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar<br>index=%{x}<br>is_correct=%{y}<br>question=%{customdata[0]}<extra></extra>",
         "legendgroup": "react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar",
         "line": {
          "color": "#FFA15A",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "react_code_gpt4o_13_aug_managedagent_noplanning_nogrammar",
         "showlegend": true,
         "type": "scattergl",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128,
          129,
          130,
          131,
          132,
          133,
          134,
          135,
          136,
          137,
          138,
          139,
          140,
          141,
          142,
          143,
          144,
          145,
          146,
          147,
          148,
          149,
          150,
          151,
          152,
          153,
          154,
          155,
          156,
          157,
          158,
          159,
          160,
          161,
          162,
          163
         ],
         "xaxis": "x",
         "y": [
          1,
          1,
          1,
          0.75,
          0.6,
          0.6666666666666666,
          0.7142857142857143,
          0.75,
          0.6666666666666666,
          0.6,
          0.5454545454545454,
          0.5,
          0.46153846153846156,
          0.42857142857142855,
          0.4,
          0.4375,
          0.47058823529411764,
          0.4444444444444444,
          0.47368421052631576,
          0.45,
          0.42857142857142855,
          0.45454545454545453,
          0.43478260869565216,
          0.4583333333333333,
          0.44,
          0.4230769230769231,
          0.4074074074074074,
          0.39285714285714285,
          0.3793103448275862,
          0.36666666666666664,
          0.3870967741935484,
          0.375,
          0.36363636363636365,
          0.38235294117647056,
          0.37142857142857144,
          0.3888888888888889,
          0.3783783783783784,
          0.3684210526315789,
          0.358974358974359,
          0.35,
          0.36585365853658536,
          0.38095238095238093,
          0.37209302325581395,
          0.36363636363636365,
          0.35555555555555557,
          0.34782608695652173,
          0.3404255319148936,
          0.3541666666666667,
          0.3673469387755102,
          0.38,
          0.37254901960784315,
          0.36538461538461536,
          0.3584905660377358,
          0.37037037037037035,
          0.36363636363636365,
          0.35714285714285715,
          0.3508771929824561,
          0.3620689655172414,
          0.3559322033898305,
          0.36666666666666664,
          0.36065573770491804,
          0.3709677419354839,
          0.38095238095238093,
          0.375,
          0.36923076923076925,
          0.36363636363636365,
          0.3582089552238806,
          0.36764705882352944,
          0.36231884057971014,
          0.35714285714285715,
          0.352112676056338,
          0.3472222222222222,
          0.3424657534246575,
          0.35135135135135137,
          0.3466666666666667,
          0.34210526315789475,
          0.33766233766233766,
          0.34615384615384615,
          0.35443037974683544,
          0.35,
          0.345679012345679,
          0.34146341463414637,
          0.3493975903614458,
          0.35714285714285715,
          0.35294117647058826,
          0.36046511627906974,
          0.367816091954023,
          0.36363636363636365,
          0.3707865168539326,
          0.36666666666666664,
          0.3626373626373626,
          0.3695652173913043,
          0.3763440860215054,
          0.3829787234042553,
          0.37894736842105264,
          0.375,
          0.3711340206185567,
          0.3673469387755102,
          0.36363636363636365,
          0.36,
          0.3564356435643564,
          0.35294117647058826,
          0.34951456310679613,
          0.34615384615384615,
          0.34285714285714286,
          0.3490566037735849,
          0.35514018691588783,
          0.3611111111111111,
          0.3669724770642202,
          0.37272727272727274,
          0.36936936936936937,
          0.36607142857142855,
          0.37168141592920356,
          0.3684210526315789,
          0.3652173913043478,
          0.3620689655172414,
          0.36752136752136755,
          0.3644067796610169,
          0.36134453781512604,
          0.36666666666666664,
          0.36363636363636365,
          0.36885245901639346,
          0.37398373983739835,
          0.3709677419354839,
          0.376,
          0.373015873015873,
          0.3779527559055118,
          0.375,
          0.37209302325581395,
          0.3769230769230769,
          0.37404580152671757,
          0.3712121212121212,
          0.3684210526315789,
          0.3656716417910448,
          0.362962962962963,
          0.36764705882352944,
          0.36496350364963503,
          0.36231884057971014,
          0.3669064748201439,
          0.36428571428571427,
          0.36879432624113473,
          0.36619718309859156,
          0.3706293706293706,
          0.375,
          0.3724137931034483,
          0.3698630136986301,
          0.3673469387755102,
          0.36486486486486486,
          0.3691275167785235,
          0.36666666666666664,
          0.3708609271523179,
          0.375,
          0.37254901960784315,
          0.37012987012987014,
          0.36774193548387096,
          0.36538461538461536,
          0.36942675159235666,
          0.37341772151898733,
          0.3710691823899371,
          0.36875,
          0.36645962732919257,
          0.37037037037037035,
          0.36809815950920244,
          0.36585365853658536
         ],
         "yaxis": "y"
        },
        {
         "customdata": [
          [
           "A paper about AI regulation that was originally su"
          ],
          [
           "I’m researching species that became invasive after"
          ],
          [
           "If we assume all articles published by Nature in 2"
          ],
          [
           "In Unlambda, what exact charcter or text needs to "
          ],
          [
           "If Eliud Kipchoge could maintain his record-making"
          ],
          [
           "The attached spreadsheet shows the inventory for a"
          ],
          [
           "How many studio albums were published by Mercedes "
          ],
          [
           "The object in the British Museum's collection with"
          ],
          [
           "According to github, when was Regression added to "
          ],
          [
           "Here's a fun riddle that I think you'll enjoy.\n\nYo"
          ],
          [
           "In July 2, 1959 United States standards for grades"
          ],
          [
           "Using the Biopython library in Python, parse the P"
          ],
          [
           "What are the EC numbers of the two most commonly u"
          ],
          [
           "In April of 1977, who was the Prime Minister of th"
          ],
          [
           "What's the last line of the rhyme under the flavor"
          ],
          [
           "Use density measures from the chemistry materials "
          ],
          [
           "What was the volume in m^3 of the fish bag that wa"
          ],
          [
           "What is the average number of pre-2020 works on th"
          ],
          [
           "In the video https://www.youtube.com/watch?v=L1vXC"
          ],
          [
           "Of the authors (First M. Last) that worked on the "
          ],
          [
           "When you take the average of the standard populati"
          ],
          [
           "In Series 9, Episode 11 of Doctor Who, the Doctor "
          ],
          [
           "In terms of geographical distance between capital "
          ],
          [
           "I need to fact-check a citation. This is the citat"
          ],
          [
           "Which contributor to the version of OpenCV where s"
          ],
          [
           "What integer-rounded percentage of the total lengt"
          ],
          [
           "What two-word type of model did Manash Pratim Kash"
          ],
          [
           ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht eti"
          ],
          [
           "What is the minimum number of page links a person "
          ],
          [
           "Each cell in the attached spreadsheet represents a"
          ],
          [
           "I went to Virtue restaurant & bar in Chicago for m"
          ],
          [
           "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) "
          ],
          [
           "My family reunion is this week, and I was assigned"
          ],
          [
           "In Emily Midkiff's June 2014 article in a journal "
          ],
          [
           "In the 2018 VSCode blog post on replit.com, what w"
          ],
          [
           "In Valentina Re’s contribution to the 2017 book “W"
          ],
          [
           "The Metropolitan Museum of Art has a portrait in i"
          ],
          [
           "In Nature journal's Scientific Reports conference "
          ],
          [
           "The attached file contains a list of vendors in th"
          ],
          [
           "In the year 2022, and before December, what does \""
          ],
          [
           "Who nominated the only Featured Article on English"
          ],
          [
           "What writer is quoted by Merriam-Webster for the W"
          ],
          [
           "How many pages if the 2023 IPCC report (85 pages v"
          ],
          [
           "Given this table defining * on the set S = {a, b, "
          ],
          [
           "The following numbers function similarly to ISBN 1"
          ],
          [
           "How many images are there in the latest 2022 Lego "
          ],
          [
           "Find the value of x to the nearest tenth: Lx = (d/"
          ],
          [
           "In the endnote found in the second-to-last paragra"
          ],
          [
           "On July 15, 2008, Phys.org published an article ab"
          ],
          [
           "You are a telecommunications engineer who wants to"
          ],
          [
           "If there is anything that doesn't make sense in th"
          ],
          [
           "The work referenced in footnote 397 of Federico La"
          ],
          [
           "How many slides in this PowerPoint presentation me"
          ],
          [
           "Given $x_0 = -5$ and $f(x) = x^3 + 4x^2 - 3x + 8$,"
          ],
          [
           "You are Van Helsing, a renowned vampire hunter. A "
          ],
          [
           "Examine the video at https://www.youtube.com/watch"
          ],
          [
           "The attached file shows the locomotives in the col"
          ],
          [
           "According to wikipedia, how many Asian countries s"
          ],
          [
           "The attached spreadsheet contains the sales of men"
          ],
          [
           "Who composed the song that was performed by a roos"
          ],
          [
           "What is the surname of the equine veterinarian men"
          ],
          [
           "I'm making a grocery list for my mom, but she's a "
          ],
          [
           "What is the last word before the second chorus of "
          ],
          [
           "Hi, I'm making a pie but I could use some help wit"
          ],
          [
           "The attached image contains a Python script. Run t"
          ],
          [
           "I have the Standard plan in the image below, and I"
          ],
          [
           "The attached PDF lists accommodations in the resor"
          ],
          [
           "The year is 2022. I am at the National Air and Spa"
          ],
          [
           "It's May 2023, and I'm about to drive across the U"
          ],
          [
           "Who did the actor who played Ray in the Polish-lan"
          ],
          [
           "On the BBC Earth YouTube video of the Top 5 Sillie"
          ],
          [
           "The book with the doi 10.1353/book.24372 concerns "
          ],
          [
           "The longest-lived vertebrate is named after an isl"
          ],
          [
           "What is the final numeric output from the attached"
          ],
          [
           "During the first week of August 2015, one of the N"
          ],
          [
           "How many more blocks (also denoted as layers) in B"
          ],
          [
           "Pull out the sentence in the following 5x7 block o"
          ],
          [
           "All of the individuals who formally held the posit"
          ],
          [
           "The attached spreadsheet contains a list of books "
          ],
          [
           "According to the USGS, in what year was the Americ"
          ],
          [
           "In the NCATS PubChem compound database for Food Ad"
          ],
          [
           "An office held a Secret Santa gift exchange where "
          ],
          [
           "What is the maximum length in meters of #9 in the "
          ],
          [
           "What animals that were mentioned in both Ilias Lag"
          ],
          [
           "How many High Energy Physics - Lattice articles li"
          ],
          [
           "The photograph in the Whitney Museum of American A"
          ],
          [
           "Which of the text elements under CATEGORIES in the"
          ],
          [
           "Under DDC 633 on Bielefeld University Library's BA"
          ],
          [
           "Compute the check digit the Tropicos ID for the Or"
          ],
          [
           "What time was the Tri-Rail train that carried the "
          ],
          [
           "Could you help me out with this assignment? Our pr"
          ],
          [
           "In the fictional language of Tizin, basic sentence"
          ],
          [
           "According to Google Finance, when was the first ye"
          ],
          [
           "According to Box Office Mojo's 2020 Worldwide Box "
          ],
          [
           "The attached file shows a list of books in the col"
          ],
          [
           "I was trying to remember how well the Cheater Beat"
          ],
          [
           "As a comma separated list with no whitespace, usin"
          ],
          [
           "On a leap day before the year 2008, a joke was rem"
          ],
          [
           "What is the volume in milliliters of a system comp"
          ],
          [
           "The Latin root of the Yola word \"gimlie\" shares a "
          ],
          [
           "Using bass clef notes, what is the age of someone "
          ],
          [
           "The attached file lists accommodations in the reso"
          ],
          [
           "In the NIH translation of the original 1913 Michae"
          ],
          [
           "How many edits were made to the Wikipedia page on "
          ],
          [
           "I was referencing each of the tables in the file f"
          ],
          [
           "How many nonindigenous crocodiles were found in Fl"
          ],
          [
           "As of the 2020 census, what was the population dif"
          ],
          [
           "This is a secret message my friend gave me. It say"
          ],
          [
           "What is the area of the green polygon in the attac"
          ],
          [
           "You are given this Excel file as a map. You start "
          ],
          [
           "I thought we could try a fun word puzzle together "
          ],
          [
           "I’m thinking about selling my home, so I want to l"
          ],
          [
           "How many times was a Twitter/X post cited as a ref"
          ],
          [
           "On ScienceDirect, what is the difference to 3 deci"
          ],
          [
           "Look at the attached image. The quiz is scored as "
          ],
          [
           "Which of the fruits shown in the 2008 painting \"Em"
          ],
          [
           "In the Scikit-Learn July 2017 changelog, what othe"
          ],
          [
           "What is the latest chronological year date written"
          ],
          [
           "The YouTube channel Game Grumps began a Let’s Play"
          ],
          [
           "This spreadsheet contains a list of clients for a "
          ],
          [
           "On the DeepFruits fruit detection graph on Connect"
          ],
          [
           "Bob was invited to participate in a game show, and"
          ],
          [
           "Consider the following symbols: 𒐜  𒐐𒐚\n\nThis is a n"
          ],
          [
           "On Cornell Law School website's legal information "
          ],
          [
           "Of the cities within the United States where U.S. "
          ],
          [
           "Eva Draconis has a personal website which can be a"
          ],
          [
           "The brand that makes these harnesses the dogs are "
          ],
          [
           "According to Girls Who Code, how long did it take "
          ],
          [
           "What was the complete title of the book in which t"
          ],
          [
           "As of August 2023, who is the only winner of the U"
          ],
          [
           "How many at bats did the Yankee with the most walk"
          ],
          [
           "The cover of the August 2021 issue of Vogue shows "
          ],
          [
           "In Audre Lorde’s poem “Father Son and Holy Ghost”,"
          ],
          [
           "I'm curious about how much information is availabl"
          ],
          [
           "The attached spreadsheet lists the locomotives own"
          ],
          [
           "What is the absolute difference in tens of thousan"
          ],
          [
           "The attached file lists the locomotives owned by a"
          ],
          [
           "Hi, I was out sick from my classes on Friday, so I"
          ],
          [
           "When was a picture of St. Thomas Aquinas first add"
          ],
          [
           "A 5-man group made up of one tank, one healer, and"
          ],
          [
           "On June 6, 2023, an article by Carolyn Collins Pet"
          ],
          [
           "According to Openreview.net, at the NeurIPS 2022 C"
          ],
          [
           "If this whole pint is made up of ice cream, how ma"
          ],
          [
           "Take the gender split from the 2011 Bulgarian cens"
          ],
          [
           "What was the actual enrollment count of the clinic"
          ],
          [
           "I'd like to learn more about some popular reality "
          ],
          [
           "Where were the Vietnamese specimens described by K"
          ],
          [
           "A standard Rubik’s cube has been broken into cubes"
          ],
          [
           "What country had the least number of athletes at t"
          ],
          [
           "I read a paper about multiwavelength observations "
          ],
          [
           "Who are the pitchers with the number before and af"
          ],
          [
           "The attached Excel file contains the sales of menu"
          ],
          [
           "In the YouTube 360 VR video from March 2018 narrat"
          ],
          [
           "In NASA's Astronomy Picture of the Day on 2006 Jan"
          ],
          [
           "In the film Goldfinger, what color was the object "
          ],
          [
           "As of May 2023, how many stops are between South S"
          ],
          [
           "In the 2015 Metropolitan Museum of Art exhibition "
          ],
          [
           "At the two-minute mark in the YouTube video upload"
          ],
          [
           "Assuming scientists in the famous youtube video Th"
          ],
          [
           "What is the first name of the only Malko Competiti"
          ],
          [
           "It is 1999. Before you party like it is 1999, plea"
          ],
          [
           "How many applicants for the job in the PDF are onl"
          ],
          [
           "What percentage of the total penguin population ac"
          ],
          [
           "According to the World Bank, which countries had g"
          ],
          [
           "Review the chess position provided in the image. I"
          ]
         ],
         "hovertemplate": "agent_name=react_code_gpt4o_23-june_planning2_newprompt5<br>index=%{x}<br>is_correct=%{y}<br>question=%{customdata[0]}<extra></extra>",
         "legendgroup": "react_code_gpt4o_23-june_planning2_newprompt5",
         "line": {
          "color": "#19d3f3",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "react_code_gpt4o_23-june_planning2_newprompt5",
         "showlegend": true,
         "type": "scattergl",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128,
          129,
          130,
          131,
          132,
          133,
          134,
          135,
          136,
          137,
          138,
          139,
          140,
          141,
          142,
          143,
          144,
          145,
          146,
          147,
          148,
          149,
          150,
          151,
          152,
          153,
          154,
          155,
          156,
          157,
          158,
          159,
          160,
          161,
          162,
          163,
          164
         ],
         "xaxis": "x",
         "y": [
          1,
          1,
          1,
          0.75,
          0.8,
          0.8333333333333334,
          0.8571428571428571,
          0.875,
          0.7777777777777778,
          0.7,
          0.6363636363636364,
          0.5833333333333334,
          0.5384615384615384,
          0.5,
          0.5333333333333333,
          0.5625,
          0.5882352941176471,
          0.5555555555555556,
          0.5789473684210527,
          0.55,
          0.5238095238095238,
          0.5,
          0.5217391304347826,
          0.5416666666666666,
          0.56,
          0.5384615384615384,
          0.5555555555555556,
          0.5714285714285714,
          0.5862068965517241,
          0.6,
          0.6129032258064516,
          0.625,
          0.6363636363636364,
          0.6470588235294118,
          0.6571428571428571,
          0.6666666666666666,
          0.6756756756756757,
          0.6842105263157895,
          0.6923076923076923,
          0.675,
          0.6585365853658537,
          0.6666666666666666,
          0.6744186046511628,
          0.6818181818181818,
          0.6888888888888889,
          0.6739130434782609,
          0.6808510638297872,
          0.6666666666666666,
          0.673469387755102,
          0.66,
          0.6666666666666666,
          0.6538461538461539,
          0.660377358490566,
          0.6666666666666666,
          0.6545454545454545,
          0.6607142857142857,
          0.6666666666666666,
          0.6551724137931034,
          0.6610169491525424,
          0.65,
          0.6557377049180327,
          0.6612903225806451,
          0.6507936507936508,
          0.65625,
          0.6615384615384615,
          0.6515151515151515,
          0.6567164179104478,
          0.6617647058823529,
          0.6521739130434783,
          0.6571428571428571,
          0.6619718309859155,
          0.6527777777777778,
          0.6575342465753424,
          0.6621621621621622,
          0.6666666666666666,
          0.6710526315789473,
          0.6753246753246753,
          0.6666666666666666,
          0.6708860759493671,
          0.675,
          0.6666666666666666,
          0.6585365853658537,
          0.6506024096385542,
          0.6428571428571429,
          0.6352941176470588,
          0.627906976744186,
          0.632183908045977,
          0.625,
          0.6292134831460674,
          0.6222222222222222,
          0.6263736263736264,
          0.6195652173913043,
          0.6129032258064516,
          0.6063829787234043,
          0.6,
          0.59375,
          0.5876288659793815,
          0.5816326530612245,
          0.5757575757575758,
          0.57,
          0.5643564356435643,
          0.5588235294117647,
          0.5533980582524272,
          0.5480769230769231,
          0.5428571428571428,
          0.5377358490566038,
          0.5327102803738317,
          0.5277777777777778,
          0.5229357798165137,
          0.5181818181818182,
          0.5135135135135135,
          0.5089285714285714,
          0.504424778761062,
          0.5,
          0.4956521739130435,
          0.49137931034482757,
          0.48717948717948717,
          0.4830508474576271,
          0.4789915966386555,
          0.475,
          0.4793388429752066,
          0.47540983606557374,
          0.4715447154471545,
          0.46774193548387094,
          0.464,
          0.46825396825396826,
          0.4645669291338583,
          0.4609375,
          0.4573643410852713,
          0.46153846153846156,
          0.4580152671755725,
          0.45454545454545453,
          0.45864661654135336,
          0.4552238805970149,
          0.45925925925925926,
          0.4632352941176471,
          0.45985401459854014,
          0.463768115942029,
          0.460431654676259,
          0.45714285714285713,
          0.46099290780141844,
          0.45774647887323944,
          0.45454545454545453,
          0.4583333333333333,
          0.46206896551724136,
          0.4589041095890411,
          0.46258503401360546,
          0.4594594594594595,
          0.46308724832214765,
          0.46,
          0.45695364238410596,
          0.4605263157894737,
          0.45751633986928103,
          0.45454545454545453,
          0.45161290322580644,
          0.4551282051282051,
          0.45222929936305734,
          0.44936708860759494,
          0.44654088050314467,
          0.45,
          0.4472049689440994,
          0.4506172839506173,
          0.44785276073619634,
          0.4451219512195122,
          0.44242424242424244
         ],
         "yaxis": "y"
        },
        {
         "customdata": [
          [
           "A paper about AI regulation that was originally su"
          ],
          [
           "I’m researching species that became invasive after"
          ],
          [
           "If we assume all articles published by Nature in 2"
          ],
          [
           "In Unlambda, what exact charcter or text needs to "
          ],
          [
           "If Eliud Kipchoge could maintain his record-making"
          ],
          [
           "The attached spreadsheet shows the inventory for a"
          ],
          [
           "How many studio albums were published by Mercedes "
          ],
          [
           "The object in the British Museum's collection with"
          ],
          [
           "According to github, when was Regression added to "
          ],
          [
           "Here's a fun riddle that I think you'll enjoy.\n\nYo"
          ],
          [
           "In July 2, 1959 United States standards for grades"
          ],
          [
           "Using the Biopython library in Python, parse the P"
          ],
          [
           "What are the EC numbers of the two most commonly u"
          ],
          [
           "In April of 1977, who was the Prime Minister of th"
          ],
          [
           "What's the last line of the rhyme under the flavor"
          ],
          [
           "Use density measures from the chemistry materials "
          ],
          [
           "What was the volume in m^3 of the fish bag that wa"
          ],
          [
           "What is the average number of pre-2020 works on th"
          ],
          [
           "In the video https://www.youtube.com/watch?v=L1vXC"
          ],
          [
           "Of the authors (First M. Last) that worked on the "
          ],
          [
           "When you take the average of the standard populati"
          ],
          [
           "Assuming scientists in the famous youtube video Th"
          ],
          [
           "In Series 9, Episode 11 of Doctor Who, the Doctor "
          ],
          [
           "In terms of geographical distance between capital "
          ],
          [
           "In the NCATS PubChem compound database for Food Ad"
          ],
          [
           "I need to fact-check a citation. This is the citat"
          ],
          [
           "Which contributor to the version of OpenCV where s"
          ],
          [
           "What integer-rounded percentage of the total lengt"
          ],
          [
           "An office held a Secret Santa gift exchange where "
          ],
          [
           "What is the maximum length in meters of #9 in the "
          ],
          [
           "What two-word type of model did Manash Pratim Kash"
          ],
          [
           "What animals that were mentioned in both Ilias Lag"
          ],
          [
           "How many High Energy Physics - Lattice articles li"
          ],
          [
           "The photograph in the Whitney Museum of American A"
          ],
          [
           ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht eti"
          ],
          [
           "What is the minimum number of page links a person "
          ],
          [
           "Each cell in the attached spreadsheet represents a"
          ],
          [
           "Which of the text elements under CATEGORIES in the"
          ],
          [
           "I went to Virtue restaurant & bar in Chicago for m"
          ],
          [
           "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) "
          ],
          [
           "My family reunion is this week, and I was assigned"
          ],
          [
           "In Emily Midkiff's June 2014 article in a journal "
          ],
          [
           "It is 1999. Before you party like it is 1999, plea"
          ],
          [
           "Under DDC 633 on Bielefeld University Library's BA"
          ],
          [
           "In the 2018 VSCode blog post on replit.com, what w"
          ],
          [
           "Compute the check digit the Tropicos ID for the Or"
          ],
          [
           "What time was the Tri-Rail train that carried the "
          ],
          [
           "Could you help me out with this assignment? Our pr"
          ],
          [
           "How many applicants for the job in the PDF are onl"
          ],
          [
           "In Valentina Re’s contribution to the 2017 book “W"
          ],
          [
           "In the fictional language of Tizin, basic sentence"
          ],
          [
           "The Metropolitan Museum of Art has a portrait in i"
          ],
          [
           "In Nature journal's Scientific Reports conference "
          ],
          [
           "The attached file contains a list of vendors in th"
          ],
          [
           "According to Google Finance, when was the first ye"
          ],
          [
           "Review the chess position provided in the image. I"
          ],
          [
           "According to Box Office Mojo's 2020 Worldwide Box "
          ],
          [
           "In the year 2022, and before December, what does \""
          ],
          [
           "Who nominated the only Featured Article on English"
          ],
          [
           "What writer is quoted by Merriam-Webster for the W"
          ],
          [
           "How many pages if the 2023 IPCC report (85 pages v"
          ],
          [
           "Given this table defining * on the set S = {a, b, "
          ],
          [
           "The following numbers function similarly to ISBN 1"
          ],
          [
           "How many images are there in the latest 2022 Lego "
          ],
          [
           "The attached file shows a list of books in the col"
          ],
          [
           "I was trying to remember how well the Cheater Beat"
          ],
          [
           "As a comma separated list with no whitespace, usin"
          ],
          [
           "On a leap day before the year 2008, a joke was rem"
          ]
         ],
         "hovertemplate": "agent_name=react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags<br>index=%{x}<br>is_correct=%{y}<br>question=%{customdata[0]}<extra></extra>",
         "legendgroup": "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags",
         "line": {
          "color": "#FF6692",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags",
         "showlegend": true,
         "type": "scattergl",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67
         ],
         "xaxis": "x",
         "y": [
          0,
          0.5,
          0.6666666666666666,
          0.5,
          0.4,
          0.5,
          0.5714285714285714,
          0.625,
          0.5555555555555556,
          0.5,
          0.45454545454545453,
          0.4166666666666667,
          0.38461538461538464,
          0.35714285714285715,
          0.4,
          0.4375,
          0.47058823529411764,
          0.4444444444444444,
          0.47368421052631576,
          0.45,
          0.42857142857142855,
          0.4090909090909091,
          0.391304347826087,
          0.4166666666666667,
          0.4,
          0.4230769230769231,
          0.4444444444444444,
          0.42857142857142855,
          0.41379310344827586,
          0.4,
          0.41935483870967744,
          0.40625,
          0.3939393939393939,
          0.38235294117647056,
          0.4,
          0.4166666666666667,
          0.43243243243243246,
          0.4473684210526316,
          0.4358974358974359,
          0.425,
          0.43902439024390244,
          0.42857142857142855,
          0.4186046511627907,
          0.4090909090909091,
          0.4,
          0.391304347826087,
          0.3829787234042553,
          0.375,
          0.3877551020408163,
          0.4,
          0.39215686274509803,
          0.40384615384615385,
          0.39622641509433965,
          0.4074074074074074,
          0.4,
          0.39285714285714285,
          0.38596491228070173,
          0.39655172413793105,
          0.4067796610169492,
          0.4166666666666667,
          0.4262295081967213,
          0.43548387096774194,
          0.42857142857142855,
          0.4375,
          0.4307692307692308,
          0.42424242424242425,
          0.417910447761194,
          0.4264705882352941
         ],
         "yaxis": "y"
        },
        {
         "customdata": [
          [
           "A paper about AI regulation that was originally su"
          ],
          [
           "I’m researching species that became invasive after"
          ],
          [
           "If we assume all articles published by Nature in 2"
          ],
          [
           "In Unlambda, what exact charcter or text needs to "
          ],
          [
           "If Eliud Kipchoge could maintain his record-making"
          ],
          [
           "The attached spreadsheet shows the inventory for a"
          ],
          [
           "How many studio albums were published by Mercedes "
          ],
          [
           "The object in the British Museum's collection with"
          ],
          [
           "According to github, when was Regression added to "
          ],
          [
           "Here's a fun riddle that I think you'll enjoy.\n\nYo"
          ],
          [
           "In July 2, 1959 United States standards for grades"
          ],
          [
           "Using the Biopython library in Python, parse the P"
          ],
          [
           "What are the EC numbers of the two most commonly u"
          ],
          [
           "In April of 1977, who was the Prime Minister of th"
          ],
          [
           "What's the last line of the rhyme under the flavor"
          ],
          [
           "Use density measures from the chemistry materials "
          ],
          [
           "What was the volume in m^3 of the fish bag that wa"
          ],
          [
           "What is the average number of pre-2020 works on th"
          ],
          [
           "In the video https://www.youtube.com/watch?v=L1vXC"
          ],
          [
           "Of the authors (First M. Last) that worked on the "
          ],
          [
           "When you take the average of the standard populati"
          ],
          [
           "Assuming scientists in the famous youtube video Th"
          ],
          [
           "In Series 9, Episode 11 of Doctor Who, the Doctor "
          ],
          [
           "In terms of geographical distance between capital "
          ],
          [
           "In the NCATS PubChem compound database for Food Ad"
          ],
          [
           "I need to fact-check a citation. This is the citat"
          ],
          [
           "Which contributor to the version of OpenCV where s"
          ],
          [
           "What integer-rounded percentage of the total lengt"
          ],
          [
           "An office held a Secret Santa gift exchange where "
          ],
          [
           "What is the maximum length in meters of #9 in the "
          ],
          [
           "What two-word type of model did Manash Pratim Kash"
          ],
          [
           "What animals that were mentioned in both Ilias Lag"
          ],
          [
           "How many High Energy Physics - Lattice articles li"
          ],
          [
           "The photograph in the Whitney Museum of American A"
          ],
          [
           ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht eti"
          ],
          [
           "What is the minimum number of page links a person "
          ],
          [
           "Each cell in the attached spreadsheet represents a"
          ],
          [
           "Which of the text elements under CATEGORIES in the"
          ],
          [
           "I went to Virtue restaurant & bar in Chicago for m"
          ],
          [
           "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) "
          ],
          [
           "My family reunion is this week, and I was assigned"
          ],
          [
           "In Emily Midkiff's June 2014 article in a journal "
          ],
          [
           "It is 1999. Before you party like it is 1999, plea"
          ],
          [
           "Under DDC 633 on Bielefeld University Library's BA"
          ],
          [
           "In the 2018 VSCode blog post on replit.com, what w"
          ],
          [
           "Compute the check digit the Tropicos ID for the Or"
          ],
          [
           "What time was the Tri-Rail train that carried the "
          ],
          [
           "Could you help me out with this assignment? Our pr"
          ],
          [
           "How many applicants for the job in the PDF are onl"
          ],
          [
           "In Valentina Re’s contribution to the 2017 book “W"
          ],
          [
           "In the fictional language of Tizin, basic sentence"
          ],
          [
           "The Metropolitan Museum of Art has a portrait in i"
          ],
          [
           "In Nature journal's Scientific Reports conference "
          ],
          [
           "The attached file contains a list of vendors in th"
          ],
          [
           "According to Google Finance, when was the first ye"
          ],
          [
           "Review the chess position provided in the image. I"
          ],
          [
           "According to Box Office Mojo's 2020 Worldwide Box "
          ],
          [
           "In the year 2022, and before December, what does \""
          ],
          [
           "Who nominated the only Featured Article on English"
          ],
          [
           "What writer is quoted by Merriam-Webster for the W"
          ],
          [
           "How many pages if the 2023 IPCC report (85 pages v"
          ],
          [
           "Given this table defining * on the set S = {a, b, "
          ],
          [
           "The following numbers function similarly to ISBN 1"
          ],
          [
           "How many images are there in the latest 2022 Lego "
          ],
          [
           "The attached file shows a list of books in the col"
          ],
          [
           "I was trying to remember how well the Cheater Beat"
          ],
          [
           "As a comma separated list with no whitespace, usin"
          ],
          [
           "On a leap day before the year 2008, a joke was rem"
          ],
          [
           "What is the volume in milliliters of a system comp"
          ],
          [
           "The Latin root of the Yola word \"gimlie\" shares a "
          ],
          [
           "Find the value of x to the nearest tenth: Lx = (d/"
          ],
          [
           "In the endnote found in the second-to-last paragra"
          ],
          [
           "Using bass clef notes, what is the age of someone "
          ],
          [
           "On July 15, 2008, Phys.org published an article ab"
          ],
          [
           "The attached file lists accommodations in the reso"
          ],
          [
           "In the NIH translation of the original 1913 Michae"
          ],
          [
           "How many edits were made to the Wikipedia page on "
          ],
          [
           "You are a telecommunications engineer who wants to"
          ],
          [
           "If there is anything that doesn't make sense in th"
          ],
          [
           "I was referencing each of the tables in the file f"
          ],
          [
           "How many nonindigenous crocodiles were found in Fl"
          ],
          [
           "The work referenced in footnote 397 of Federico La"
          ],
          [
           "As of the 2020 census, what was the population dif"
          ],
          [
           "How many slides in this PowerPoint presentation me"
          ],
          [
           "What percentage of the total penguin population ac"
          ],
          [
           "Given $x_0 = -5$ and $f(x) = x^3 + 4x^2 - 3x + 8$,"
          ],
          [
           "You are Van Helsing, a renowned vampire hunter. A "
          ],
          [
           "Examine the video at https://www.youtube.com/watch"
          ],
          [
           "The attached file shows the locomotives in the col"
          ],
          [
           "This is a secret message my friend gave me. It say"
          ],
          [
           "What is the area of the green polygon in the attac"
          ],
          [
           "According to wikipedia, how many Asian countries s"
          ],
          [
           "The attached spreadsheet contains the sales of men"
          ],
          [
           "Who composed the song that was performed by a roos"
          ],
          [
           "You are given this Excel file as a map. You start "
          ],
          [
           "I thought we could try a fun word puzzle together "
          ],
          [
           "What is the surname of the equine veterinarian men"
          ],
          [
           "According to the World Bank, which countries had g"
          ],
          [
           "I’m thinking about selling my home, so I want to l"
          ],
          [
           "I'm making a grocery list for my mom, but she's a "
          ]
         ],
         "hovertemplate": "agent_name=react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch<br>index=%{x}<br>is_correct=%{y}<br>question=%{customdata[0]}<extra></extra>",
         "legendgroup": "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch",
         "line": {
          "color": "#B6E880",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "react_code_gpt4o_31_aug_defaultplanning_nogrammar_normaltags_nonavigationalsearch",
         "showlegend": true,
         "type": "scattergl",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99
         ],
         "xaxis": "x",
         "y": [
          0,
          0,
          0,
          0,
          0.2,
          0.3333333333333333,
          0.42857142857142855,
          0.5,
          0.4444444444444444,
          0.4,
          0.36363636363636365,
          0.3333333333333333,
          0.3076923076923077,
          0.2857142857142857,
          0.26666666666666666,
          0.25,
          0.29411764705882354,
          0.2777777777777778,
          0.2631578947368421,
          0.25,
          0.23809523809523808,
          0.2727272727272727,
          0.2608695652173913,
          0.2916666666666667,
          0.28,
          0.3076923076923077,
          0.2962962962962963,
          0.2857142857142857,
          0.27586206896551724,
          0.26666666666666666,
          0.2903225806451613,
          0.28125,
          0.2727272727272727,
          0.2647058823529412,
          0.2857142857142857,
          0.2777777777777778,
          0.2702702702702703,
          0.2631578947368421,
          0.28205128205128205,
          0.275,
          0.2926829268292683,
          0.30952380952380953,
          0.3023255813953488,
          0.29545454545454547,
          0.3111111111111111,
          0.32608695652173914,
          0.3191489361702128,
          0.3125,
          0.32653061224489793,
          0.34,
          0.3333333333333333,
          0.34615384615384615,
          0.3584905660377358,
          0.35185185185185186,
          0.34545454545454546,
          0.3392857142857143,
          0.3333333333333333,
          0.3275862068965517,
          0.3220338983050847,
          0.3333333333333333,
          0.32786885245901637,
          0.3387096774193548,
          0.3333333333333333,
          0.328125,
          0.3230769230769231,
          0.3181818181818182,
          0.31343283582089554,
          0.3088235294117647,
          0.30434782608695654,
          0.3,
          0.29577464788732394,
          0.2916666666666667,
          0.2876712328767123,
          0.2972972972972973,
          0.29333333333333333,
          0.2894736842105263,
          0.2857142857142857,
          0.28205128205128205,
          0.2911392405063291,
          0.2875,
          0.2839506172839506,
          0.2926829268292683,
          0.2891566265060241,
          0.2976190476190476,
          0.29411764705882354,
          0.29069767441860467,
          0.28735632183908044,
          0.29545454545454547,
          0.30337078651685395,
          0.3,
          0.2967032967032967,
          0.29347826086956524,
          0.3010752688172043,
          0.30851063829787234,
          0.30526315789473685,
          0.3020833333333333,
          0.29896907216494845,
          0.29591836734693877,
          0.29292929292929293,
          0.29
         ],
         "yaxis": "y"
        },
        {
         "customdata": [
          [
           "A paper about AI regulation that was originally su"
          ],
          [
           "I’m researching species that became invasive after"
          ],
          [
           "If we assume all articles published by Nature in 2"
          ],
          [
           "In Unlambda, what exact charcter or text needs to "
          ],
          [
           "If Eliud Kipchoge could maintain his record-making"
          ],
          [
           "The attached spreadsheet shows the inventory for a"
          ],
          [
           "How many studio albums were published by Mercedes "
          ],
          [
           "The object in the British Museum's collection with"
          ],
          [
           "According to github, when was Regression added to "
          ],
          [
           "Here's a fun riddle that I think you'll enjoy.\n\nYo"
          ],
          [
           "In July 2, 1959 United States standards for grades"
          ],
          [
           "Using the Biopython library in Python, parse the P"
          ],
          [
           "What are the EC numbers of the two most commonly u"
          ],
          [
           "In April of 1977, who was the Prime Minister of th"
          ],
          [
           "What's the last line of the rhyme under the flavor"
          ],
          [
           "Use density measures from the chemistry materials "
          ],
          [
           "What was the volume in m^3 of the fish bag that wa"
          ],
          [
           "What is the average number of pre-2020 works on th"
          ],
          [
           "In the video https://www.youtube.com/watch?v=L1vXC"
          ],
          [
           "Of the authors (First M. Last) that worked on the "
          ],
          [
           "When you take the average of the standard populati"
          ],
          [
           "Assuming scientists in the famous youtube video Th"
          ],
          [
           "In Series 9, Episode 11 of Doctor Who, the Doctor "
          ],
          [
           "In terms of geographical distance between capital "
          ],
          [
           "In the NCATS PubChem compound database for Food Ad"
          ],
          [
           "I need to fact-check a citation. This is the citat"
          ],
          [
           "Which contributor to the version of OpenCV where s"
          ],
          [
           "What integer-rounded percentage of the total lengt"
          ],
          [
           "An office held a Secret Santa gift exchange where "
          ],
          [
           "What is the maximum length in meters of #9 in the "
          ],
          [
           "What two-word type of model did Manash Pratim Kash"
          ],
          [
           "What animals that were mentioned in both Ilias Lag"
          ],
          [
           "How many High Energy Physics - Lattice articles li"
          ],
          [
           "The photograph in the Whitney Museum of American A"
          ],
          [
           ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht eti"
          ],
          [
           "What is the minimum number of page links a person "
          ],
          [
           "Each cell in the attached spreadsheet represents a"
          ],
          [
           "Which of the text elements under CATEGORIES in the"
          ],
          [
           "I went to Virtue restaurant & bar in Chicago for m"
          ],
          [
           "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) "
          ],
          [
           "My family reunion is this week, and I was assigned"
          ],
          [
           "In Emily Midkiff's June 2014 article in a journal "
          ],
          [
           "It is 1999. Before you party like it is 1999, plea"
          ],
          [
           "Under DDC 633 on Bielefeld University Library's BA"
          ],
          [
           "In the 2018 VSCode blog post on replit.com, what w"
          ],
          [
           "Compute the check digit the Tropicos ID for the Or"
          ],
          [
           "What time was the Tri-Rail train that carried the "
          ],
          [
           "Could you help me out with this assignment? Our pr"
          ],
          [
           "How many applicants for the job in the PDF are onl"
          ],
          [
           "In Valentina Re’s contribution to the 2017 book “W"
          ],
          [
           "In the fictional language of Tizin, basic sentence"
          ],
          [
           "The Metropolitan Museum of Art has a portrait in i"
          ],
          [
           "In Nature journal's Scientific Reports conference "
          ],
          [
           "The attached file contains a list of vendors in th"
          ],
          [
           "According to Google Finance, when was the first ye"
          ],
          [
           "Review the chess position provided in the image. I"
          ],
          [
           "According to Box Office Mojo's 2020 Worldwide Box "
          ],
          [
           "In the year 2022, and before December, what does \""
          ],
          [
           "Who nominated the only Featured Article on English"
          ],
          [
           "What writer is quoted by Merriam-Webster for the W"
          ],
          [
           "How many pages if the 2023 IPCC report (85 pages v"
          ],
          [
           "Given this table defining * on the set S = {a, b, "
          ],
          [
           "The following numbers function similarly to ISBN 1"
          ],
          [
           "How many images are there in the latest 2022 Lego "
          ],
          [
           "The attached file shows a list of books in the col"
          ],
          [
           "I was trying to remember how well the Cheater Beat"
          ],
          [
           "As a comma separated list with no whitespace, usin"
          ],
          [
           "On a leap day before the year 2008, a joke was rem"
          ],
          [
           "What is the volume in milliliters of a system comp"
          ],
          [
           "The Latin root of the Yola word \"gimlie\" shares a "
          ],
          [
           "Find the value of x to the nearest tenth: Lx = (d/"
          ],
          [
           "In the endnote found in the second-to-last paragra"
          ],
          [
           "Using bass clef notes, what is the age of someone "
          ],
          [
           "On July 15, 2008, Phys.org published an article ab"
          ],
          [
           "The attached file lists accommodations in the reso"
          ],
          [
           "In the NIH translation of the original 1913 Michae"
          ],
          [
           "How many edits were made to the Wikipedia page on "
          ],
          [
           "You are a telecommunications engineer who wants to"
          ],
          [
           "If there is anything that doesn't make sense in th"
          ],
          [
           "I was referencing each of the tables in the file f"
          ],
          [
           "How many nonindigenous crocodiles were found in Fl"
          ],
          [
           "The work referenced in footnote 397 of Federico La"
          ],
          [
           "As of the 2020 census, what was the population dif"
          ],
          [
           "How many slides in this PowerPoint presentation me"
          ],
          [
           "What percentage of the total penguin population ac"
          ],
          [
           "Given $x_0 = -5$ and $f(x) = x^3 + 4x^2 - 3x + 8$,"
          ],
          [
           "You are Van Helsing, a renowned vampire hunter. A "
          ],
          [
           "Examine the video at https://www.youtube.com/watch"
          ],
          [
           "The attached file shows the locomotives in the col"
          ],
          [
           "This is a secret message my friend gave me. It say"
          ],
          [
           "What is the area of the green polygon in the attac"
          ],
          [
           "According to wikipedia, how many Asian countries s"
          ],
          [
           "The attached spreadsheet contains the sales of men"
          ],
          [
           "Who composed the song that was performed by a roos"
          ],
          [
           "You are given this Excel file as a map. You start "
          ],
          [
           "I thought we could try a fun word puzzle together "
          ],
          [
           "What is the surname of the equine veterinarian men"
          ],
          [
           "According to the World Bank, which countries had g"
          ],
          [
           "I’m thinking about selling my home, so I want to l"
          ],
          [
           "I'm making a grocery list for my mom, but she's a "
          ],
          [
           "How many times was a Twitter/X post cited as a ref"
          ],
          [
           "On ScienceDirect, what is the difference to 3 deci"
          ],
          [
           "What is the last word before the second chorus of "
          ],
          [
           "Look at the attached image. The quiz is scored as "
          ],
          [
           "Which of the fruits shown in the 2008 painting \"Em"
          ],
          [
           "Hi, I'm making a pie but I could use some help wit"
          ],
          [
           "The attached image contains a Python script. Run t"
          ],
          [
           "I have the Standard plan in the image below, and I"
          ],
          [
           "The attached PDF lists accommodations in the resor"
          ],
          [
           "The year is 2022. I am at the National Air and Spa"
          ],
          [
           "In the Scikit-Learn July 2017 changelog, what othe"
          ],
          [
           "It's May 2023, and I'm about to drive across the U"
          ],
          [
           "Who did the actor who played Ray in the Polish-lan"
          ],
          [
           "What is the latest chronological year date written"
          ],
          [
           "The YouTube channel Game Grumps began a Let’s Play"
          ],
          [
           "This spreadsheet contains a list of clients for a "
          ],
          [
           "On the BBC Earth YouTube video of the Top 5 Sillie"
          ],
          [
           "The book with the doi 10.1353/book.24372 concerns "
          ],
          [
           "The longest-lived vertebrate is named after an isl"
          ],
          [
           "What is the final numeric output from the attached"
          ],
          [
           "On the DeepFruits fruit detection graph on Connect"
          ],
          [
           "During the first week of August 2015, one of the N"
          ],
          [
           "How many more blocks (also denoted as layers) in B"
          ],
          [
           "Bob was invited to participate in a game show, and"
          ],
          [
           "Pull out the sentence in the following 5x7 block o"
          ],
          [
           "All of the individuals who formally held the posit"
          ],
          [
           "The attached spreadsheet contains a list of books "
          ],
          [
           "Consider the following symbols: 𒐜  𒐐𒐚\n\nThis is a n"
          ],
          [
           "On Cornell Law School website's legal information "
          ],
          [
           "According to the USGS, in what year was the Americ"
          ],
          [
           "Of the cities within the United States where U.S. "
          ],
          [
           "Eva Draconis has a personal website which can be a"
          ],
          [
           "The brand that makes these harnesses the dogs are "
          ],
          [
           "According to Girls Who Code, how long did it take "
          ],
          [
           "What was the complete title of the book in which t"
          ]
         ],
         "hovertemplate": "agent_name=react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags<br>index=%{x}<br>is_correct=%{y}<br>question=%{customdata[0]}<extra></extra>",
         "legendgroup": "react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags",
         "line": {
          "color": "#FF97FF",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "react_code_gpt4o_31_aug_structuredplanning_nogrammar_htmltags",
         "showlegend": true,
         "type": "scattergl",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128,
          129,
          130,
          131,
          132,
          133,
          134
         ],
         "xaxis": "x",
         "y": [
          1,
          1,
          0.6666666666666666,
          0.5,
          0.4,
          0.5,
          0.5714285714285714,
          0.625,
          0.5555555555555556,
          0.5,
          0.45454545454545453,
          0.4166666666666667,
          0.38461538461538464,
          0.35714285714285715,
          0.3333333333333333,
          0.375,
          0.35294117647058826,
          0.3333333333333333,
          0.3157894736842105,
          0.3,
          0.2857142857142857,
          0.2727272727272727,
          0.2608695652173913,
          0.2916666666666667,
          0.28,
          0.2692307692307692,
          0.25925925925925924,
          0.2857142857142857,
          0.27586206896551724,
          0.26666666666666666,
          0.2903225806451613,
          0.28125,
          0.2727272727272727,
          0.2647058823529412,
          0.2857142857142857,
          0.2777777777777778,
          0.2702702702702703,
          0.2631578947368421,
          0.2564102564102564,
          0.25,
          0.2682926829268293,
          0.2857142857142857,
          0.27906976744186046,
          0.2727272727272727,
          0.26666666666666666,
          0.2608695652173913,
          0.2553191489361702,
          0.25,
          0.24489795918367346,
          0.26,
          0.2549019607843137,
          0.25,
          0.24528301886792453,
          0.25925925925925924,
          0.2545454545454545,
          0.25,
          0.24561403508771928,
          0.25862068965517243,
          0.2711864406779661,
          0.2833333333333333,
          0.2786885245901639,
          0.2903225806451613,
          0.30158730158730157,
          0.296875,
          0.2923076923076923,
          0.2878787878787879,
          0.2835820895522388,
          0.29411764705882354,
          0.2898550724637681,
          0.2857142857142857,
          0.29577464788732394,
          0.2916666666666667,
          0.2876712328767123,
          0.2972972972972973,
          0.29333333333333333,
          0.2894736842105263,
          0.2987012987012987,
          0.2948717948717949,
          0.3037974683544304,
          0.3,
          0.2962962962962963,
          0.2926829268292683,
          0.2891566265060241,
          0.2976190476190476,
          0.29411764705882354,
          0.3023255813953488,
          0.2988505747126437,
          0.3068181818181818,
          0.3146067415730337,
          0.3111111111111111,
          0.3076923076923077,
          0.30434782608695654,
          0.3118279569892473,
          0.3191489361702128,
          0.3157894736842105,
          0.3125,
          0.31958762886597936,
          0.3163265306122449,
          0.31313131313131315,
          0.31,
          0.3069306930693069,
          0.30392156862745096,
          0.3106796116504854,
          0.3076923076923077,
          0.3047619047619048,
          0.3113207547169811,
          0.3177570093457944,
          0.32407407407407407,
          0.3211009174311927,
          0.3181818181818182,
          0.3153153153153153,
          0.3125,
          0.3185840707964602,
          0.3157894736842105,
          0.3130434782608696,
          0.3103448275862069,
          0.3162393162393162,
          0.3220338983050847,
          0.3277310924369748,
          0.3333333333333333,
          0.3305785123966942,
          0.3360655737704918,
          0.34146341463414637,
          0.3387096774193548,
          0.344,
          0.3412698412698413,
          0.3464566929133858,
          0.34375,
          0.34108527131782945,
          0.3384615384615385,
          0.33587786259541985,
          0.3333333333333333,
          0.3308270676691729,
          0.3283582089552239,
          0.3333333333333333
         ],
         "yaxis": "y"
        },
        {
         "customdata": [
          [
           "A paper about AI regulation that was originally su"
          ],
          [
           "I’m researching species that became invasive after"
          ],
          [
           "If we assume all articles published by Nature in 2"
          ],
          [
           "In Unlambda, what exact charcter or text needs to "
          ],
          [
           "If Eliud Kipchoge could maintain his record-making"
          ],
          [
           "The attached spreadsheet shows the inventory for a"
          ],
          [
           "How many studio albums were published by Mercedes "
          ],
          [
           "The object in the British Museum's collection with"
          ],
          [
           "According to github, when was Regression added to "
          ],
          [
           "Here's a fun riddle that I think you'll enjoy.\n\nYo"
          ],
          [
           "In July 2, 1959 United States standards for grades"
          ],
          [
           "Using the Biopython library in Python, parse the P"
          ],
          [
           "What are the EC numbers of the two most commonly u"
          ],
          [
           "In April of 1977, who was the Prime Minister of th"
          ],
          [
           "What's the last line of the rhyme under the flavor"
          ],
          [
           "Use density measures from the chemistry materials "
          ],
          [
           "What was the volume in m^3 of the fish bag that wa"
          ],
          [
           "What is the average number of pre-2020 works on th"
          ],
          [
           "In the video https://www.youtube.com/watch?v=L1vXC"
          ],
          [
           "Of the authors (First M. Last) that worked on the "
          ],
          [
           "When you take the average of the standard populati"
          ],
          [
           "Assuming scientists in the famous youtube video Th"
          ],
          [
           "In Series 9, Episode 11 of Doctor Who, the Doctor "
          ],
          [
           "In terms of geographical distance between capital "
          ],
          [
           "In the NCATS PubChem compound database for Food Ad"
          ],
          [
           "I need to fact-check a citation. This is the citat"
          ],
          [
           "Which contributor to the version of OpenCV where s"
          ],
          [
           "What integer-rounded percentage of the total lengt"
          ],
          [
           "An office held a Secret Santa gift exchange where "
          ],
          [
           "What is the maximum length in meters of #9 in the "
          ],
          [
           "What two-word type of model did Manash Pratim Kash"
          ],
          [
           "What animals that were mentioned in both Ilias Lag"
          ],
          [
           "How many High Energy Physics - Lattice articles li"
          ],
          [
           "The photograph in the Whitney Museum of American A"
          ],
          [
           ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht eti"
          ],
          [
           "What is the minimum number of page links a person "
          ],
          [
           "Each cell in the attached spreadsheet represents a"
          ],
          [
           "Which of the text elements under CATEGORIES in the"
          ],
          [
           "I went to Virtue restaurant & bar in Chicago for m"
          ],
          [
           "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) "
          ],
          [
           "My family reunion is this week, and I was assigned"
          ],
          [
           "In Emily Midkiff's June 2014 article in a journal "
          ],
          [
           "It is 1999. Before you party like it is 1999, plea"
          ],
          [
           "Under DDC 633 on Bielefeld University Library's BA"
          ],
          [
           "In the 2018 VSCode blog post on replit.com, what w"
          ],
          [
           "Compute the check digit the Tropicos ID for the Or"
          ],
          [
           "What time was the Tri-Rail train that carried the "
          ],
          [
           "Could you help me out with this assignment? Our pr"
          ],
          [
           "How many applicants for the job in the PDF are onl"
          ],
          [
           "In Valentina Re’s contribution to the 2017 book “W"
          ],
          [
           "In the fictional language of Tizin, basic sentence"
          ],
          [
           "The Metropolitan Museum of Art has a portrait in i"
          ],
          [
           "In Nature journal's Scientific Reports conference "
          ],
          [
           "The attached file contains a list of vendors in th"
          ],
          [
           "According to Google Finance, when was the first ye"
          ],
          [
           "Review the chess position provided in the image. I"
          ],
          [
           "According to Box Office Mojo's 2020 Worldwide Box "
          ],
          [
           "In the year 2022, and before December, what does \""
          ],
          [
           "Who nominated the only Featured Article on English"
          ],
          [
           "What writer is quoted by Merriam-Webster for the W"
          ],
          [
           "How many pages if the 2023 IPCC report (85 pages v"
          ],
          [
           "Given this table defining * on the set S = {a, b, "
          ],
          [
           "The following numbers function similarly to ISBN 1"
          ],
          [
           "How many images are there in the latest 2022 Lego "
          ],
          [
           "The attached file shows a list of books in the col"
          ],
          [
           "I was trying to remember how well the Cheater Beat"
          ],
          [
           "As a comma separated list with no whitespace, usin"
          ],
          [
           "On a leap day before the year 2008, a joke was rem"
          ],
          [
           "What is the volume in milliliters of a system comp"
          ],
          [
           "The Latin root of the Yola word \"gimlie\" shares a "
          ],
          [
           "Find the value of x to the nearest tenth: Lx = (d/"
          ],
          [
           "In the endnote found in the second-to-last paragra"
          ],
          [
           "Using bass clef notes, what is the age of someone "
          ],
          [
           "On July 15, 2008, Phys.org published an article ab"
          ],
          [
           "The attached file lists accommodations in the reso"
          ],
          [
           "In the NIH translation of the original 1913 Michae"
          ],
          [
           "How many edits were made to the Wikipedia page on "
          ],
          [
           "You are a telecommunications engineer who wants to"
          ],
          [
           "If there is anything that doesn't make sense in th"
          ],
          [
           "I was referencing each of the tables in the file f"
          ],
          [
           "How many nonindigenous crocodiles were found in Fl"
          ],
          [
           "The work referenced in footnote 397 of Federico La"
          ],
          [
           "As of the 2020 census, what was the population dif"
          ],
          [
           "How many slides in this PowerPoint presentation me"
          ],
          [
           "What percentage of the total penguin population ac"
          ],
          [
           "Given $x_0 = -5$ and $f(x) = x^3 + 4x^2 - 3x + 8$,"
          ],
          [
           "You are Van Helsing, a renowned vampire hunter. A "
          ],
          [
           "Examine the video at https://www.youtube.com/watch"
          ],
          [
           "The attached file shows the locomotives in the col"
          ],
          [
           "This is a secret message my friend gave me. It say"
          ],
          [
           "What is the area of the green polygon in the attac"
          ],
          [
           "According to wikipedia, how many Asian countries s"
          ],
          [
           "The attached spreadsheet contains the sales of men"
          ],
          [
           "Who composed the song that was performed by a roos"
          ],
          [
           "You are given this Excel file as a map. You start "
          ],
          [
           "I thought we could try a fun word puzzle together "
          ],
          [
           "What is the surname of the equine veterinarian men"
          ],
          [
           "According to the World Bank, which countries had g"
          ],
          [
           "I’m thinking about selling my home, so I want to l"
          ],
          [
           "I'm making a grocery list for my mom, but she's a "
          ],
          [
           "How many times was a Twitter/X post cited as a ref"
          ],
          [
           "On ScienceDirect, what is the difference to 3 deci"
          ],
          [
           "What is the last word before the second chorus of "
          ],
          [
           "Look at the attached image. The quiz is scored as "
          ],
          [
           "Which of the fruits shown in the 2008 painting \"Em"
          ],
          [
           "Hi, I'm making a pie but I could use some help wit"
          ],
          [
           "The attached image contains a Python script. Run t"
          ],
          [
           "I have the Standard plan in the image below, and I"
          ],
          [
           "The attached PDF lists accommodations in the resor"
          ],
          [
           "The year is 2022. I am at the National Air and Spa"
          ],
          [
           "In the Scikit-Learn July 2017 changelog, what othe"
          ],
          [
           "It's May 2023, and I'm about to drive across the U"
          ],
          [
           "Who did the actor who played Ray in the Polish-lan"
          ],
          [
           "What is the latest chronological year date written"
          ],
          [
           "The YouTube channel Game Grumps began a Let’s Play"
          ],
          [
           "This spreadsheet contains a list of clients for a "
          ],
          [
           "On the BBC Earth YouTube video of the Top 5 Sillie"
          ],
          [
           "The book with the doi 10.1353/book.24372 concerns "
          ],
          [
           "The longest-lived vertebrate is named after an isl"
          ],
          [
           "What is the final numeric output from the attached"
          ],
          [
           "On the DeepFruits fruit detection graph on Connect"
          ],
          [
           "During the first week of August 2015, one of the N"
          ],
          [
           "How many more blocks (also denoted as layers) in B"
          ],
          [
           "Bob was invited to participate in a game show, and"
          ],
          [
           "Pull out the sentence in the following 5x7 block o"
          ],
          [
           "All of the individuals who formally held the posit"
          ],
          [
           "The attached spreadsheet contains a list of books "
          ],
          [
           "Consider the following symbols: 𒐜  𒐐𒐚\n\nThis is a n"
          ],
          [
           "On Cornell Law School website's legal information "
          ],
          [
           "According to the USGS, in what year was the Americ"
          ],
          [
           "Of the cities within the United States where U.S. "
          ],
          [
           "Eva Draconis has a personal website which can be a"
          ],
          [
           "The brand that makes these harnesses the dogs are "
          ],
          [
           "According to Girls Who Code, how long did it take "
          ],
          [
           "What was the complete title of the book in which t"
          ],
          [
           "As of August 2023, who is the only winner of the U"
          ],
          [
           "How many at bats did the Yankee with the most walk"
          ],
          [
           "The cover of the August 2021 issue of Vogue shows "
          ],
          [
           "In Audre Lorde’s poem “Father Son and Holy Ghost”,"
          ],
          [
           "I'm curious about how much information is availabl"
          ],
          [
           "The attached spreadsheet lists the locomotives own"
          ],
          [
           "What is the absolute difference in tens of thousan"
          ],
          [
           "The attached file lists the locomotives owned by a"
          ],
          [
           "Hi, I was out sick from my classes on Friday, so I"
          ],
          [
           "When was a picture of St. Thomas Aquinas first add"
          ],
          [
           "A 5-man group made up of one tank, one healer, and"
          ],
          [
           "On June 6, 2023, an article by Carolyn Collins Pet"
          ],
          [
           "According to Openreview.net, at the NeurIPS 2022 C"
          ],
          [
           "Take the gender split from the 2011 Bulgarian cens"
          ],
          [
           "What was the actual enrollment count of the clinic"
          ],
          [
           "I'd like to learn more about some popular reality "
          ],
          [
           "Where were the Vietnamese specimens described by K"
          ],
          [
           "A standard Rubik’s cube has been broken into cubes"
          ],
          [
           "What country had the least number of athletes at t"
          ],
          [
           "I read a paper about multiwavelength observations "
          ],
          [
           "Who are the pitchers with the number before and af"
          ],
          [
           "The attached Excel file contains the sales of menu"
          ],
          [
           "What is the first name of the only Malko Competiti"
          ],
          [
           "In the YouTube 360 VR video from March 2018 narrat"
          ],
          [
           "In NASA's Astronomy Picture of the Day on 2006 Jan"
          ],
          [
           "In the film Goldfinger, what color was the object "
          ],
          [
           "As of May 2023, how many stops are between South S"
          ],
          [
           "In the 2015 Metropolitan Museum of Art exhibition "
          ],
          [
           "At the two-minute mark in the YouTube video upload"
          ]
         ],
         "hovertemplate": "agent_name=react_code_gpt4o_6_aug_planning_nogrammar<br>index=%{x}<br>is_correct=%{y}<br>question=%{customdata[0]}<extra></extra>",
         "legendgroup": "react_code_gpt4o_6_aug_planning_nogrammar",
         "line": {
          "color": "#FECB52",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "react_code_gpt4o_6_aug_planning_nogrammar",
         "showlegend": true,
         "type": "scattergl",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128,
          129,
          130,
          131,
          132,
          133,
          134,
          135,
          136,
          137,
          138,
          139,
          140,
          141,
          142,
          143,
          144,
          145,
          146,
          147,
          148,
          149,
          150,
          151,
          152,
          153,
          154,
          155,
          156,
          157,
          158,
          159,
          160,
          161,
          162,
          163
         ],
         "xaxis": "x",
         "y": [
          0,
          0.5,
          0.3333333333333333,
          0.25,
          0.4,
          0.5,
          0.5714285714285714,
          0.625,
          0.5555555555555556,
          0.5,
          0.45454545454545453,
          0.4166666666666667,
          0.38461538461538464,
          0.35714285714285715,
          0.3333333333333333,
          0.375,
          0.4117647058823529,
          0.3888888888888889,
          0.3684210526315789,
          0.35,
          0.3333333333333333,
          0.3181818181818182,
          0.30434782608695654,
          0.3333333333333333,
          0.32,
          0.34615384615384615,
          0.37037037037037035,
          0.35714285714285715,
          0.3448275862068966,
          0.3333333333333333,
          0.3548387096774194,
          0.34375,
          0.3333333333333333,
          0.3235294117647059,
          0.34285714285714286,
          0.3611111111111111,
          0.35135135135135137,
          0.3684210526315789,
          0.38461538461538464,
          0.4,
          0.4146341463414634,
          0.42857142857142855,
          0.4418604651162791,
          0.4318181818181818,
          0.4222222222222222,
          0.41304347826086957,
          0.40425531914893614,
          0.3958333333333333,
          0.3877551020408163,
          0.4,
          0.39215686274509803,
          0.40384615384615385,
          0.39622641509433965,
          0.4074074074074074,
          0.4,
          0.39285714285714285,
          0.38596491228070173,
          0.3793103448275862,
          0.3898305084745763,
          0.4,
          0.4098360655737705,
          0.41935483870967744,
          0.42857142857142855,
          0.421875,
          0.4153846153846154,
          0.4090909090909091,
          0.40298507462686567,
          0.4117647058823529,
          0.4057971014492754,
          0.4,
          0.4084507042253521,
          0.4027777777777778,
          0.3972602739726027,
          0.40540540540540543,
          0.4,
          0.39473684210526316,
          0.38961038961038963,
          0.38461538461538464,
          0.3924050632911392,
          0.3875,
          0.38271604938271603,
          0.3902439024390244,
          0.3855421686746988,
          0.38095238095238093,
          0.3764705882352941,
          0.37209302325581395,
          0.367816091954023,
          0.375,
          0.38202247191011235,
          0.37777777777777777,
          0.37362637362637363,
          0.3695652173913043,
          0.3763440860215054,
          0.3829787234042553,
          0.37894736842105264,
          0.375,
          0.3711340206185567,
          0.3673469387755102,
          0.36363636363636365,
          0.36,
          0.3564356435643564,
          0.35294117647058826,
          0.34951456310679613,
          0.34615384615384615,
          0.34285714285714286,
          0.3490566037735849,
          0.35514018691588783,
          0.3611111111111111,
          0.3669724770642202,
          0.37272727272727274,
          0.36936936936936937,
          0.375,
          0.3805309734513274,
          0.37719298245614036,
          0.3826086956521739,
          0.3793103448275862,
          0.38461538461538464,
          0.3813559322033898,
          0.3865546218487395,
          0.39166666666666666,
          0.3884297520661157,
          0.39344262295081966,
          0.3983739837398374,
          0.3951612903225806,
          0.4,
          0.3968253968253968,
          0.4015748031496063,
          0.3984375,
          0.3953488372093023,
          0.3923076923076923,
          0.3893129770992366,
          0.38636363636363635,
          0.38345864661654133,
          0.3805970149253731,
          0.3851851851851852,
          0.3897058823529412,
          0.38686131386861317,
          0.38405797101449274,
          0.381294964028777,
          0.37857142857142856,
          0.375886524822695,
          0.38028169014084506,
          0.38461538461538464,
          0.3888888888888889,
          0.38620689655172413,
          0.3835616438356164,
          0.38095238095238093,
          0.3783783783783784,
          0.3825503355704698,
          0.38,
          0.37748344370860926,
          0.3815789473684211,
          0.3790849673202614,
          0.38311688311688313,
          0.38064516129032255,
          0.3782051282051282,
          0.37579617834394907,
          0.379746835443038,
          0.37735849056603776,
          0.375,
          0.37267080745341613,
          0.37037037037037035,
          0.36809815950920244,
          0.36585365853658536
         ],
         "yaxis": "y"
        },
        {
         "customdata": [
          [
           "A paper about AI regulation that was originally su"
          ],
          [
           "I’m researching species that became invasive after"
          ],
          [
           "If we assume all articles published by Nature in 2"
          ],
          [
           "In Unlambda, what exact charcter or text needs to "
          ],
          [
           "If Eliud Kipchoge could maintain his record-making"
          ],
          [
           "The attached spreadsheet shows the inventory for a"
          ],
          [
           "How many studio albums were published by Mercedes "
          ],
          [
           "According to github, when was Regression added to "
          ],
          [
           "Here's a fun riddle that I think you'll enjoy.\n\nYo"
          ],
          [
           "In July 2, 1959 United States standards for grades"
          ],
          [
           "What are the EC numbers of the two most commonly u"
          ],
          [
           "Use density measures from the chemistry materials "
          ],
          [
           "What was the volume in m^3 of the fish bag that wa"
          ],
          [
           "When you take the average of the standard populati"
          ],
          [
           "In Series 9, Episode 11 of Doctor Who, the Doctor "
          ],
          [
           "In terms of geographical distance between capital "
          ],
          [
           "In the NCATS PubChem compound database for Food Ad"
          ],
          [
           "I need to fact-check a citation. This is the citat"
          ],
          [
           "Which contributor to the version of OpenCV where s"
          ],
          [
           "What integer-rounded percentage of the total lengt"
          ],
          [
           "An office held a Secret Santa gift exchange where "
          ],
          [
           "What is the maximum length in meters of #9 in the "
          ],
          [
           "What two-word type of model did Manash Pratim Kash"
          ],
          [
           "What animals that were mentioned in both Ilias Lag"
          ],
          [
           "How many High Energy Physics - Lattice articles li"
          ],
          [
           "The photograph in the Whitney Museum of American A"
          ],
          [
           ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht eti"
          ],
          [
           "What is the minimum number of page links a person "
          ],
          [
           "Each cell in the attached spreadsheet represents a"
          ],
          [
           "Which of the text elements under CATEGORIES in the"
          ],
          [
           "I went to Virtue restaurant & bar in Chicago for m"
          ],
          [
           "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) "
          ],
          [
           "My family reunion is this week, and I was assigned"
          ],
          [
           "Under DDC 633 on Bielefeld University Library's BA"
          ],
          [
           "Could you help me out with this assignment? Our pr"
          ],
          [
           "How many applicants for the job in the PDF are onl"
          ],
          [
           "In Valentina Re’s contribution to the 2017 book “W"
          ],
          [
           "In the fictional language of Tizin, basic sentence"
          ],
          [
           "The Metropolitan Museum of Art has a portrait in i"
          ],
          [
           "In Nature journal's Scientific Reports conference "
          ],
          [
           "The attached file contains a list of vendors in th"
          ],
          [
           "Review the chess position provided in the image. I"
          ],
          [
           "According to Box Office Mojo's 2020 Worldwide Box "
          ],
          [
           "The object in the British Museum's collection with"
          ],
          [
           "Using the Biopython library in Python, parse the P"
          ],
          [
           "In April of 1977, who was the Prime Minister of th"
          ],
          [
           "What's the last line of the rhyme under the flavor"
          ],
          [
           "What is the average number of pre-2020 works on th"
          ],
          [
           "In the video https://www.youtube.com/watch?v=L1vXC"
          ],
          [
           "Of the authors (First M. Last) that worked on the "
          ],
          [
           "Assuming scientists in the famous youtube video Th"
          ],
          [
           "In Emily Midkiff's June 2014 article in a journal "
          ],
          [
           "It is 1999. Before you party like it is 1999, plea"
          ],
          [
           "In the 2018 VSCode blog post on replit.com, what w"
          ],
          [
           "Compute the check digit the Tropicos ID for the Or"
          ],
          [
           "What time was the Tri-Rail train that carried the "
          ],
          [
           "According to Google Finance, when was the first ye"
          ],
          [
           "In the year 2022, and before December, what does \""
          ]
         ],
         "hovertemplate": "agent_name=react_code_llama-31-70B_06_sept_managedagent-summary_planning<br>index=%{x}<br>is_correct=%{y}<br>question=%{customdata[0]}<extra></extra>",
         "legendgroup": "react_code_llama-31-70B_06_sept_managedagent-summary_planning",
         "line": {
          "color": "#636efa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "react_code_llama-31-70B_06_sept_managedagent-summary_planning",
         "showlegend": true,
         "type": "scattergl",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57
         ],
         "xaxis": "x",
         "y": [
          1,
          0.5,
          0.3333333333333333,
          0.25,
          0.2,
          0.3333333333333333,
          0.2857142857142857,
          0.25,
          0.2222222222222222,
          0.2,
          0.18181818181818182,
          0.25,
          0.3076923076923077,
          0.2857142857142857,
          0.26666666666666666,
          0.3125,
          0.29411764705882354,
          0.2777777777777778,
          0.2631578947368421,
          0.25,
          0.23809523809523808,
          0.22727272727272727,
          0.21739130434782608,
          0.20833333333333334,
          0.2,
          0.19230769230769232,
          0.18518518518518517,
          0.17857142857142858,
          0.1724137931034483,
          0.2,
          0.1935483870967742,
          0.1875,
          0.18181818181818182,
          0.17647058823529413,
          0.17142857142857143,
          0.16666666666666666,
          0.1891891891891892,
          0.18421052631578946,
          0.20512820512820512,
          0.2,
          0.1951219512195122,
          0.19047619047619047,
          0.20930232558139536,
          0.20454545454545456,
          0.2,
          0.1956521739130435,
          0.19148936170212766,
          0.1875,
          0.1836734693877551,
          0.18,
          0.17647058823529413,
          0.17307692307692307,
          0.16981132075471697,
          0.16666666666666666,
          0.18181818181818182,
          0.17857142857142858,
          0.17543859649122806,
          0.1724137931034483
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "legend": {
         "title": {
          "text": "agent_name"
         },
         "tracegroupgap": 0
        },
        "margin": {
         "t": 60
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "index"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "is_correct"
         }
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import plotly.express as px\n",
    "\n",
    "cumulative_df = (\n",
    "    (\n",
    "        sel_df.groupby(\"agent_name\")[[\"is_correct\", \"is_near_correct\"]]\n",
    "        .expanding(min_periods=1, axis=0, method=\"single\")\n",
    "        .agg({\"is_correct\": \"mean\", \"is_near_correct\": \"count\"})\n",
    "        .reset_index()\n",
    "    )\n",
    "    .copy()\n",
    "    .rename(columns={\"is_near_correct\": \"index\"})\n",
    ")\n",
    "cumulative_df[\"index\"] = cumulative_df[\"index\"].astype(int) - 1\n",
    "\n",
    "\n",
    "def find_question(row):\n",
    "    try:\n",
    "        res = sel_df.loc[sel_df[\"agent_name\"] == row[\"agent_name\"], \"question\"].iloc[\n",
    "            row[\"index\"]\n",
    "        ][:50]\n",
    "        return res\n",
    "    except Exception as e:\n",
    "        return \"\"\n",
    "\n",
    "\n",
    "cumulative_df[\"question\"] = cumulative_df.apply(find_question, axis=1)\n",
    "# cumulative_df[\"question\"] = [el[:50] for el in sel_df[\"question\"].values]\n",
    "\n",
    "# cumulative_df[\"is_correct\"] = cumulative_df[\"is_correct\"] * (165 - 68) / 165\n",
    "\n",
    "px.line(\n",
    "    cumulative_df,\n",
    "    color=\"agent_name\",\n",
    "    x=\"index\",\n",
    "    y=\"is_correct\",\n",
    "    hover_data=\"question\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Dive deeper into one run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'gpt4o_multiagent_summary' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[26], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m sel_df \u001b[38;5;241m=\u001b[39m result_df\u001b[38;5;241m.\u001b[39mloc[result_df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_name\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m==\u001b[39m \u001b[43mgpt4o_multiagent_summary\u001b[49m]\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;28mlen\u001b[39m(sel_df))\n",
      "\u001b[0;31mNameError\u001b[0m: name 'gpt4o_multiagent_summary' is not defined"
     ]
    }
   ],
   "source": [
    "sel_df = result_df.loc[result_df[\"agent_name\"] == gpt4o_multiagent_summary]\n",
    "print(len(sel_df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/6m/9b1tts6d5w960j80wbw9tx3m0000gn/T/ipykernel_87644/2783934181.py:9: SettingWithCopyWarning:\n",
      "\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "/var/folders/6m/9b1tts6d5w960j80wbw9tx3m0000gn/T/ipykernel_87644/2783934181.py:9: SettingWithCopyWarning:\n",
      "\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "/var/folders/6m/9b1tts6d5w960j80wbw9tx3m0000gn/T/ipykernel_87644/2783934181.py:9: SettingWithCopyWarning:\n",
      "\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "/var/folders/6m/9b1tts6d5w960j80wbw9tx3m0000gn/T/ipykernel_87644/2783934181.py:9: SettingWithCopyWarning:\n",
      "\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "/var/folders/6m/9b1tts6d5w960j80wbw9tx3m0000gn/T/ipykernel_87644/2783934181.py:10: SettingWithCopyWarning:\n",
      "\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "error_types = [\n",
    "    \"AgentParsingError\",\n",
    "    \"AgentExecutionError\",\n",
    "    \"AgentMaxIterationsError\",\n",
    "    \"AgentGenerationError\",\n",
    "]\n",
    "sel_df[error_types] = 0\n",
    "sel_df[\"Count steps\"] = np.nan\n",
    "\n",
    "\n",
    "def count_errors(row):\n",
    "    if isinstance(row[\"intermediate_steps\"], list):\n",
    "        row[\"Count steps\"] = len(row[\"intermediate_steps\"])\n",
    "        for step in row[\"intermediate_steps\"]:\n",
    "            if isinstance(step, dict) and \"error\" in step:\n",
    "                try:\n",
    "                    row[str(step[\"error\"][\"error_type\"])] += 1\n",
    "                except:\n",
    "                    pass\n",
    "    return row\n",
    "\n",
    "\n",
    "sel_df = sel_df.apply(count_errors, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "alignmentgroup": "True",
         "hovertemplate": "is_correct=False<br>variable=%{x}<br><b>Average count</b>=%{y}<extra></extra>",
         "legendgroup": "False",
         "marker": {
          "color": "#636efa",
          "pattern": {
           "shape": ""
          }
         },
         "name": "False",
         "offsetgroup": "False",
         "orientation": "v",
         "showlegend": true,
         "textposition": "outside",
         "type": "bar",
         "x": [
          "AgentParsingError",
          "AgentExecutionError",
          "AgentMaxIterationsError",
          "AgentGenerationError",
          "Count steps"
         ],
         "xaxis": "x",
         "y": [
          0.34210526315789475,
          0.7894736842105263,
          0.42105263157894735,
          0,
          12.223684210526315
         ],
         "yaxis": "y"
        },
        {
         "alignmentgroup": "True",
         "hovertemplate": "is_correct=True<br>variable=%{x}<br><b>Average count</b>=%{y}<extra></extra>",
         "legendgroup": "True",
         "marker": {
          "color": "#EF553B",
          "pattern": {
           "shape": ""
          }
         },
         "name": "True",
         "offsetgroup": "True",
         "orientation": "v",
         "showlegend": true,
         "textposition": "outside",
         "type": "bar",
         "x": [
          "AgentParsingError",
          "AgentExecutionError",
          "AgentMaxIterationsError",
          "AgentGenerationError",
          "Count steps"
         ],
         "xaxis": "x",
         "y": [
          0.16666666666666666,
          0.6666666666666666,
          0.08333333333333333,
          0,
          7.416666666666667
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "bargroupgap": 0,
        "barmode": "group",
        "height": 500,
        "legend": {
         "title": {
          "text": "is_correct"
         },
         "tracegroupgap": 0
        },
        "margin": {
         "t": 60
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "width": 800,
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "variable"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "<b>Average count</b>"
         }
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import plotly.express as px\n",
    "\n",
    "\n",
    "aggregate_errors = (\n",
    "    sel_df.groupby([\"is_correct\"])[error_types + [\"Count steps\"]]\n",
    "    .mean()\n",
    "    .reset_index()\n",
    "    .melt(id_vars=[\"is_correct\"])\n",
    ")\n",
    "\n",
    "fig = px.bar(\n",
    "    aggregate_errors,\n",
    "    y=\"value\",\n",
    "    x=\"variable\",\n",
    "    color=\"is_correct\",\n",
    "    labels={\n",
    "        \"agent_name\": \"<b>LLM Engine</b>\",\n",
    "        \"task\": \"<b>Level</b>\",\n",
    "        \"aggregate_score\": \"<b>Performance</b>\",\n",
    "        \"value\": \"<b>Average count</b>\",\n",
    "        \"eval_score_GPT4\": \"<b>Score</b>\",\n",
    "    },\n",
    ")\n",
    "fig.update_layout(\n",
    "    height=500,\n",
    "    width=800,\n",
    "    barmode=\"group\",\n",
    "    bargroupgap=0.0,\n",
    ")\n",
    "fig.update_traces(textposition=\"outside\")\n",
    "fig.write_image(\"figures/aggregate_errors.png\", scale=3)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count tool calls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>print</th>\n",
       "      <th>ask_search_agent</th>\n",
       "      <th>final_answer</th>\n",
       "      <th>len</th>\n",
       "      <th>range</th>\n",
       "      <th>inspect_file_as_text</th>\n",
       "      <th>set</th>\n",
       "      <th>visualizer</th>\n",
       "      <th>parse_square</th>\n",
       "      <th>sum</th>\n",
       "      <th>...</th>\n",
       "      <th>max</th>\n",
       "      <th>join</th>\n",
       "      <th>generate_prefixes</th>\n",
       "      <th>sorted</th>\n",
       "      <th>get</th>\n",
       "      <th>lower</th>\n",
       "      <th>f</th>\n",
       "      <th>search_birthdate</th>\n",
       "      <th>items</th>\n",
       "      <th>abs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>20.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     print  ask_search_agent  final_answer  len  range  inspect_file_as_text  \\\n",
       "0      8.0               3.0           1.0  0.0    0.0                   5.0   \n",
       "1      3.0               2.0           1.0  0.0    0.0                   0.0   \n",
       "2      5.0               5.0           1.0  0.0    0.0                   0.0   \n",
       "3      4.0               3.0           1.0  0.0    0.0                   0.0   \n",
       "4      3.0               2.0           1.0  0.0    0.0                   0.0   \n",
       "..     ...               ...           ...  ...    ...                   ...   \n",
       "160    5.0               5.0           1.0  0.0    0.0                   0.0   \n",
       "161    3.0               0.0           1.0  1.0    0.0                   2.0   \n",
       "162    7.0               2.0           1.0  4.0    0.0                   0.0   \n",
       "163   20.0               8.0           0.0  0.0    1.0                   2.0   \n",
       "164    2.0               0.0           1.0  0.0    0.0                   0.0   \n",
       "\n",
       "     set  visualizer  parse_square  sum  ...  max  join  generate_prefixes  \\\n",
       "0    0.0         0.0           0.0  0.0  ...  0.0   0.0                0.0   \n",
       "1    0.0         0.0           0.0  0.0  ...  0.0   0.0                0.0   \n",
       "2    0.0         0.0           0.0  0.0  ...  0.0   0.0                0.0   \n",
       "3    0.0         0.0           0.0  0.0  ...  0.0   0.0                0.0   \n",
       "4    0.0         0.0           0.0  0.0  ...  0.0   0.0                0.0   \n",
       "..   ...         ...           ...  ...  ...  ...   ...                ...   \n",
       "160  0.0         0.0           0.0  0.0  ...  0.0   0.0                0.0   \n",
       "161  0.0         0.0           0.0  0.0  ...  0.0   0.0                0.0   \n",
       "162  0.0         0.0           0.0  0.0  ...  0.0   0.0                0.0   \n",
       "163  0.0         0.0           0.0  0.0  ...  0.0   0.0                0.0   \n",
       "164  0.0         1.0          36.0  0.0  ...  0.0   0.0                0.0   \n",
       "\n",
       "     sorted  get  lower    f  search_birthdate  items  abs  \n",
       "0       0.0  0.0    0.0  0.0               0.0    0.0  0.0  \n",
       "1       0.0  0.0    0.0  0.0               0.0    0.0  0.0  \n",
       "2       0.0  0.0    0.0  0.0               0.0    0.0  0.0  \n",
       "3       0.0  0.0    0.0  0.0               0.0    0.0  0.0  \n",
       "4       0.0  0.0    0.0  0.0               0.0    0.0  0.0  \n",
       "..      ...  ...    ...  ...               ...    ...  ...  \n",
       "160     0.0  0.0    0.0  0.0               0.0    0.0  0.0  \n",
       "161     0.0  0.0    0.0  0.0               0.0    0.0  0.0  \n",
       "162     0.0  0.0    0.0  0.0               0.0    0.0  0.0  \n",
       "163     0.0  0.0    0.0  0.0               0.0    0.0  0.0  \n",
       "164     0.0  0.0    0.0  0.0               0.0    0.0  0.0  \n",
       "\n",
       "[165 rows x 31 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tools_calls = pd.DataFrame.from_records(sel_df[\"tool_calls\"].values).fillna(0)\n",
    "\n",
    "# Exclude the tools that were not used enough\n",
    "tools_calls = tools_calls.loc[:, tools_calls.sum() > 10]\n",
    "\n",
    "# Sort the columns by the sum of the values\n",
    "tools_calls = tools_calls[tools_calls.sum().sort_values(ascending=False).index]\n",
    "display(tools_calls)\n",
    "sel_with_calls = pd.concat(\n",
    "    [sel_df[[\"question\", \"is_correct\", \"task\"]], tools_calls], axis=1\n",
    ")\n",
    "sel_with_calls = (\n",
    "    sel_with_calls.drop(\"question\", axis=1).groupby([\"is_correct\", \"task\"]).mean()\n",
    ")\n",
    "# sel_with_calls = sel_with_calls.melt(id_vars=['question', 'is_correct', 'task'], var_name=\"tool\", value_name='count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "sel_with_calls = sel_with_calls.reset_index().melt(\n",
    "    id_vars=[\"is_correct\", \"task\"], var_name=\"tool\", value_name=\"average_count\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "alignmentgroup": "True",
         "hovertemplate": "is_correct=False<br><b>Level</b>=1<br>tool=%{x}<br><b>Average #calls per run</b>=%{y}<extra></extra>",
         "legendgroup": "False",
         "marker": {
          "color": "#636efa",
          "pattern": {
           "shape": ""
          }
         },
         "name": "False",
         "offsetgroup": "False",
         "orientation": "v",
         "showlegend": true,
         "textposition": "auto",
         "type": "bar",
         "x": [
          "print",
          "ask_search_agent",
          "final_answer",
          "len",
          "range",
          "inspect_file_as_text",
          "set",
          "visualizer",
          "parse_square",
          "sum",
          "append",
          "round",
          "dfs",
          "pop",
          "split",
          "list",
          "set_piece_at",
          "add",
          "piece_at",
          "is_valid",
          "find_words",
          "max",
          "join",
          "generate_prefixes",
          "sorted",
          "get",
          "lower",
          "f",
          "search_birthdate",
          "items",
          "abs"
         ],
         "xaxis": "x3",
         "y": [
          3.3181818181818183,
          1.8181818181818181,
          1.1363636363636365,
          0.4090909090909091,
          0.6363636363636364,
          0.13636363636363635,
          0.2727272727272727,
          0.2727272727272727,
          1.6363636363636365,
          0.045454545454545456,
          0.8181818181818182,
          0,
          0,
          0.9545454545454546,
          0,
          0.22727272727272727,
          0.8181818181818182,
          0,
          0.8181818181818182,
          0,
          0,
          0.2727272727272727,
          0.09090909090909091,
          0,
          0.045454545454545456,
          0,
          0,
          0,
          0,
          0.2727272727272727,
          0.13636363636363635
         ],
         "yaxis": "y3"
        },
        {
         "alignmentgroup": "True",
         "hovertemplate": "is_correct=False<br><b>Level</b>=2<br>tool=%{x}<br><b>Average #calls per run</b>=%{y}<extra></extra>",
         "legendgroup": "False",
         "marker": {
          "color": "#636efa",
          "pattern": {
           "shape": ""
          }
         },
         "name": "False",
         "offsetgroup": "False",
         "orientation": "v",
         "showlegend": false,
         "textposition": "auto",
         "type": "bar",
         "x": [
          "print",
          "ask_search_agent",
          "final_answer",
          "len",
          "range",
          "inspect_file_as_text",
          "set",
          "visualizer",
          "parse_square",
          "sum",
          "append",
          "round",
          "dfs",
          "pop",
          "split",
          "list",
          "set_piece_at",
          "add",
          "piece_at",
          "is_valid",
          "find_words",
          "max",
          "join",
          "generate_prefixes",
          "sorted",
          "get",
          "lower",
          "f",
          "search_birthdate",
          "items",
          "abs"
         ],
         "xaxis": "x2",
         "y": [
          5.122448979591836,
          3.306122448979592,
          0.8571428571428571,
          0.42857142857142855,
          0.061224489795918366,
          0.2857142857142857,
          0,
          0.24489795918367346,
          0,
          0.2653061224489796,
          0.20408163265306123,
          0.22448979591836735,
          0,
          0,
          0.16326530612244897,
          0,
          0,
          0,
          0,
          0,
          0,
          0.04081632653061224,
          0.02040816326530612,
          0,
          0,
          0.22448979591836735,
          0.12244897959183673,
          0,
          0,
          0.02040816326530612,
          0.10204081632653061
         ],
         "yaxis": "y2"
        },
        {
         "alignmentgroup": "True",
         "hovertemplate": "is_correct=False<br><b>Level</b>=3<br>tool=%{x}<br><b>Average #calls per run</b>=%{y}<extra></extra>",
         "legendgroup": "False",
         "marker": {
          "color": "#636efa",
          "pattern": {
           "shape": ""
          }
         },
         "name": "False",
         "offsetgroup": "False",
         "orientation": "v",
         "showlegend": false,
         "textposition": "auto",
         "type": "bar",
         "x": [
          "print",
          "ask_search_agent",
          "final_answer",
          "len",
          "range",
          "inspect_file_as_text",
          "set",
          "visualizer",
          "parse_square",
          "sum",
          "append",
          "round",
          "dfs",
          "pop",
          "split",
          "list",
          "set_piece_at",
          "add",
          "piece_at",
          "is_valid",
          "find_words",
          "max",
          "join",
          "generate_prefixes",
          "sorted",
          "get",
          "lower",
          "f",
          "search_birthdate",
          "items",
          "abs"
         ],
         "xaxis": "x",
         "y": [
          8.714285714285714,
          4.857142857142857,
          0.8095238095238095,
          2.238095238095238,
          1.9047619047619047,
          0.6190476190476191,
          1.5238095238095237,
          0.23809523809523808,
          0,
          0,
          0.09523809523809523,
          0.2857142857142857,
          1.1428571428571428,
          0,
          0.047619047619047616,
          0.2857142857142857,
          0,
          0.7142857142857143,
          0,
          0.7619047619047619,
          0.7619047619047619,
          0.2857142857142857,
          0.09523809523809523,
          0.6666666666666666,
          0.14285714285714285,
          0,
          0,
          0,
          0,
          0,
          0
         ],
         "yaxis": "y"
        },
        {
         "alignmentgroup": "True",
         "hovertemplate": "is_correct=True<br><b>Level</b>=1<br>tool=%{x}<br><b>Average #calls per run</b>=%{y}<extra></extra>",
         "legendgroup": "True",
         "marker": {
          "color": "#EF553B",
          "pattern": {
           "shape": ""
          }
         },
         "name": "True",
         "offsetgroup": "True",
         "orientation": "v",
         "showlegend": true,
         "textposition": "auto",
         "type": "bar",
         "x": [
          "print",
          "ask_search_agent",
          "final_answer",
          "len",
          "range",
          "inspect_file_as_text",
          "set",
          "visualizer",
          "parse_square",
          "sum",
          "append",
          "round",
          "dfs",
          "pop",
          "split",
          "list",
          "set_piece_at",
          "add",
          "piece_at",
          "is_valid",
          "find_words",
          "max",
          "join",
          "generate_prefixes",
          "sorted",
          "get",
          "lower",
          "f",
          "search_birthdate",
          "items",
          "abs"
         ],
         "xaxis": "x3",
         "y": [
          2.4838709677419355,
          1.5161290322580645,
          1.032258064516129,
          0.06451612903225806,
          0.06451612903225806,
          0.3548387096774194,
          0.06451612903225806,
          0.03225806451612903,
          0,
          0.1935483870967742,
          0.03225806451612903,
          0.03225806451612903,
          0.06451612903225806,
          0.03225806451612903,
          0.0967741935483871,
          0.03225806451612903,
          0,
          0.0967741935483871,
          0,
          0,
          0,
          0,
          0.1935483870967742,
          0,
          0.06451612903225806,
          0,
          0,
          0,
          0,
          0.03225806451612903,
          0
         ],
         "yaxis": "y3"
        },
        {
         "alignmentgroup": "True",
         "hovertemplate": "is_correct=True<br><b>Level</b>=2<br>tool=%{x}<br><b>Average #calls per run</b>=%{y}<extra></extra>",
         "legendgroup": "True",
         "marker": {
          "color": "#EF553B",
          "pattern": {
           "shape": ""
          }
         },
         "name": "True",
         "offsetgroup": "True",
         "orientation": "v",
         "showlegend": false,
         "textposition": "auto",
         "type": "bar",
         "x": [
          "print",
          "ask_search_agent",
          "final_answer",
          "len",
          "range",
          "inspect_file_as_text",
          "set",
          "visualizer",
          "parse_square",
          "sum",
          "append",
          "round",
          "dfs",
          "pop",
          "split",
          "list",
          "set_piece_at",
          "add",
          "piece_at",
          "is_valid",
          "find_words",
          "max",
          "join",
          "generate_prefixes",
          "sorted",
          "get",
          "lower",
          "f",
          "search_birthdate",
          "items",
          "abs"
         ],
         "xaxis": "x2",
         "y": [
          5.162162162162162,
          2.7027027027027026,
          0.9459459459459459,
          0.10810810810810811,
          0.10810810810810811,
          0.32432432432432434,
          0.2972972972972973,
          0.32432432432432434,
          0,
          0.21621621621621623,
          0,
          0.21621621621621623,
          0,
          0,
          0.13513513513513514,
          0.16216216216216217,
          0,
          0,
          0,
          0,
          0,
          0.02702702702702703,
          0.02702702702702703,
          0,
          0.21621621621621623,
          0,
          0.16216216216216217,
          0.32432432432432434,
          0.32432432432432434,
          0.05405405405405406,
          0.08108108108108109
         ],
         "yaxis": "y2"
        },
        {
         "alignmentgroup": "True",
         "hovertemplate": "is_correct=True<br><b>Level</b>=3<br>tool=%{x}<br><b>Average #calls per run</b>=%{y}<extra></extra>",
         "legendgroup": "True",
         "marker": {
          "color": "#EF553B",
          "pattern": {
           "shape": ""
          }
         },
         "name": "True",
         "offsetgroup": "True",
         "orientation": "v",
         "showlegend": false,
         "textposition": "auto",
         "type": "bar",
         "x": [
          "print",
          "ask_search_agent",
          "final_answer",
          "len",
          "range",
          "inspect_file_as_text",
          "set",
          "visualizer",
          "parse_square",
          "sum",
          "append",
          "round",
          "dfs",
          "pop",
          "split",
          "list",
          "set_piece_at",
          "add",
          "piece_at",
          "is_valid",
          "find_words",
          "max",
          "join",
          "generate_prefixes",
          "sorted",
          "get",
          "lower",
          "f",
          "search_birthdate",
          "items",
          "abs"
         ],
         "xaxis": "x",
         "y": [
          6.4,
          2.2,
          0.8,
          0,
          0.4,
          1.6,
          0.2,
          0.2,
          0,
          0.8,
          0.2,
          0,
          0,
          0,
          0.8,
          0.4,
          0,
          0,
          0,
          0,
          0,
          0,
          0.4,
          0,
          0,
          0.2,
          0,
          0,
          0,
          0.4,
          0
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "annotations": [
         {
          "font": {},
          "showarrow": false,
          "text": "<b>Level</b>=3",
          "textangle": 90,
          "x": 0.98,
          "xanchor": "left",
          "xref": "paper",
          "y": 0.15666666666666665,
          "yanchor": "middle",
          "yref": "paper"
         },
         {
          "font": {},
          "showarrow": false,
          "text": "<b>Level</b>=2",
          "textangle": 90,
          "x": 0.98,
          "xanchor": "left",
          "xref": "paper",
          "y": 0.4999999999999999,
          "yanchor": "middle",
          "yref": "paper"
         },
         {
          "font": {},
          "showarrow": false,
          "text": "<b>Level</b>=1",
          "textangle": 90,
          "x": 0.98,
          "xanchor": "left",
          "xref": "paper",
          "y": 0.8433333333333332,
          "yanchor": "middle",
          "yref": "paper"
         }
        ],
        "barmode": "group",
        "height": 800,
        "legend": {
         "title": {
          "text": "is_correct"
         },
         "tracegroupgap": 0
        },
        "margin": {
         "t": 60
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "<b></b>"
        },
        "width": 1000,
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          0.98
         ],
         "title": {
          "text": "tool"
         }
        },
        "xaxis2": {
         "anchor": "y2",
         "domain": [
          0,
          0.98
         ],
         "matches": "x",
         "showticklabels": false
        },
        "xaxis3": {
         "anchor": "y3",
         "domain": [
          0,
          0.98
         ],
         "matches": "x",
         "showticklabels": false
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          0.3133333333333333
         ],
         "title": {
          "text": "<b>Average #calls per run</b>"
         }
        },
        "yaxis2": {
         "anchor": "x2",
         "domain": [
          0.34333333333333327,
          0.6566666666666665
         ],
         "matches": "y",
         "title": {
          "text": "<b>Average #calls per run</b>"
         }
        },
        "yaxis3": {
         "anchor": "x3",
         "domain": [
          0.6866666666666665,
          0.9999999999999998
         ],
         "matches": "y",
         "title": {
          "text": "<b>Average #calls per run</b>"
         }
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import plotly.express as px\n",
    "\n",
    "fig = px.bar(\n",
    "    sel_with_calls,\n",
    "    x=\"tool\",\n",
    "    y=\"average_count\",\n",
    "    color=\"is_correct\",\n",
    "    facet_row=\"task\",\n",
    "    labels={\n",
    "        \"agent_name\": \"<b>Agent variant</b>\",\n",
    "        \"task\": \"<b>Level</b>\",\n",
    "        \"aggregate_score\": \"<b>Performance</b>\",\n",
    "        \"eval_score_GPT4\": \"<b>Score</b>\",\n",
    "        \"agent_type\": \"<b>Agent type</b>\",\n",
    "        \"average_count\": \"<b>Average #calls per run</b>\",\n",
    "    },\n",
    ")\n",
    "fig.update_layout(\n",
    "    barmode=\"group\",\n",
    "    height=800,\n",
    "    width=1000,\n",
    "    title=\"<b>\" + \"</b>\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inspect result by file extension type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>is_correct</th>\n",
       "      <th>count_steps</th>\n",
       "      <th>question</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>agent_name</th>\n",
       "      <th>attachment_type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"14\" valign=\"top\">react_code_gpt4o_23-june_planning2_newprompt5</th>\n",
       "      <th>None</th>\n",
       "      <td>0.440945</td>\n",
       "      <td>9.196850</td>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>csv</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>docx</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jpg</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.500000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jsonld</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mp3</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.333333</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pdb</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pdf</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>5.666667</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>png</th>\n",
       "      <td>0.125000</td>\n",
       "      <td>6.750000</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pptx</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>py</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>txt</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xlsx</th>\n",
       "      <td>0.615385</td>\n",
       "      <td>7.538462</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zip</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                               is_correct  \\\n",
       "agent_name                                    attachment_type               \n",
       "react_code_gpt4o_23-june_planning2_newprompt5 None               0.440945   \n",
       "                                              csv                0.000000   \n",
       "                                              docx               0.000000   \n",
       "                                              jpg                0.000000   \n",
       "                                              jsonld             0.000000   \n",
       "                                              mp3                1.000000   \n",
       "                                              pdb                0.000000   \n",
       "                                              pdf                0.333333   \n",
       "                                              png                0.125000   \n",
       "                                              pptx               1.000000   \n",
       "                                              py                 1.000000   \n",
       "                                              txt                0.000000   \n",
       "                                              xlsx               0.615385   \n",
       "                                              zip                1.000000   \n",
       "\n",
       "                                                               count_steps  \\\n",
       "agent_name                                    attachment_type                \n",
       "react_code_gpt4o_23-june_planning2_newprompt5 None                9.196850   \n",
       "                                              csv                 7.000000   \n",
       "                                              docx                9.000000   \n",
       "                                              jpg                 9.500000   \n",
       "                                              jsonld             16.000000   \n",
       "                                              mp3                 8.333333   \n",
       "                                              pdb                 7.000000   \n",
       "                                              pdf                 5.666667   \n",
       "                                              png                 6.750000   \n",
       "                                              pptx                4.000000   \n",
       "                                              py                  4.000000   \n",
       "                                              txt                 6.000000   \n",
       "                                              xlsx                7.538462   \n",
       "                                              zip                10.000000   \n",
       "\n",
       "                                                               question  \n",
       "agent_name                                    attachment_type            \n",
       "react_code_gpt4o_23-june_planning2_newprompt5 None                  127  \n",
       "                                              csv                     1  \n",
       "                                              docx                    1  \n",
       "                                              jpg                     2  \n",
       "                                              jsonld                  1  \n",
       "                                              mp3                     3  \n",
       "                                              pdb                     1  \n",
       "                                              pdf                     3  \n",
       "                                              png                     8  \n",
       "                                              pptx                    1  \n",
       "                                              py                      1  \n",
       "                                              txt                     1  \n",
       "                                              xlsx                   13  \n",
       "                                              zip                     2  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(\n",
    "    sel_df.groupby([\"agent_name\", \"attachment_type\"])[\n",
    "        [\"is_correct\", \"count_steps\", \"question\"]\n",
    "    ].agg({\"is_correct\": \"mean\", \"count_steps\": \"mean\", \"question\": \"count\"})\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inspect specific file types\n",
    "# sel_df.loc[\n",
    "#     sel_df[\"attachment_type\"].isin([\"pdb\", \"docx\", \"csv\"]),\n",
    "#     [\n",
    "#         \"attachment_type\",\n",
    "#         \"question\",\n",
    "#         \"prediction\",\n",
    "#         \"true_answer\",\n",
    "#         \"is_correct\",\n",
    "#         \"thoughts\",\n",
    "#     ],\n",
    "# ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Ensembling methods\n",
    "\n",
    "### 4.1 Simple retry mechanism"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n",
      "replaced\n"
     ]
    }
   ],
   "source": [
    "first_run_gpt4 = result_df.loc[result_df[\"agent_name\"] == gpt4o].copy()\n",
    "second_run_gpt4 = result_df.loc[result_df[\"agent_name\"] == noanchorplan].copy()\n",
    "\n",
    "\n",
    "def replace_answer_if_incomplete(row, result_df_replacement):\n",
    "    try:\n",
    "        if (\n",
    "            \"Unable to determine\" in row[\"intermediate_steps\"]\n",
    "            or \"AgentMaxIterationsError\" in str(row[\"intermediate_steps\"])\n",
    "            # or \"AgentExecutionError\" in str(row[\"intermediate_steps\"])\n",
    "            # or \"AgentGenerationError\" in str(row[\"intermediate_steps\"])\n",
    "            or \"Error in generating final llm output\" in str(row[\"intermediate_steps\"])\n",
    "        ):\n",
    "            matching_answer = result_df_replacement.loc[\n",
    "                (result_df_replacement[\"question\"] == row[\"question\"]), \"prediction\"\n",
    "            ].values[0]\n",
    "            print(\"replaced\")\n",
    "            gold_answer = matching_answer\n",
    "        else:\n",
    "            gold_answer = row[\"prediction\"]\n",
    "    except:\n",
    "        gold_answer = row[\"prediction\"]\n",
    "    return gold_answer\n",
    "\n",
    "\n",
    "combined_gpt4 = first_run_gpt4.copy()\n",
    "combined_gpt4[\"prediction\"] = combined_gpt4.apply(\n",
    "    lambda x: replace_answer_if_incomplete(x, second_run_gpt4), axis=1\n",
    ")\n",
    "\n",
    "combined_gpt4[\"is_correct\"] = combined_gpt4.apply(\n",
    "    lambda x: question_scorer(x[\"prediction\"], x[\"true_answer\"]), axis=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First run:\n",
      "task\n",
      "1    0.566038\n",
      "2    0.418605\n",
      "3    0.200000\n",
      "Name: is_correct, dtype: float64\n",
      "0.4329268292682927\n",
      "Second run:\n",
      "task\n",
      "1    0.528302\n",
      "2    0.372093\n",
      "3    0.200000\n",
      "Name: is_correct, dtype: float64\n",
      "0.39634146341463417\n",
      "Combined run:\n",
      "task\n",
      "1    0.566038\n",
      "2    0.395349\n",
      "3    0.160000\n",
      "Name: is_correct, dtype: float64\n",
      "0.4146341463414634\n"
     ]
    }
   ],
   "source": [
    "print(\"First run:\")\n",
    "print(first_run_gpt4.groupby([\"task\"])[\"is_correct\"].mean())\n",
    "print(first_run_gpt4[\"is_correct\"].mean())\n",
    "\n",
    "print(\"Second run:\")\n",
    "print(second_run_gpt4.groupby([\"task\"])[\"is_correct\"].mean())\n",
    "print(second_run_gpt4[\"is_correct\"].mean())\n",
    "\n",
    "print(\"Combined run:\")\n",
    "print(combined_gpt4.groupby([\"task\"])[\"is_correct\"].mean())\n",
    "print(combined_gpt4[\"is_correct\"].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 Ideal ensembling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ideal combined run:\n",
      "task\n",
      "1    0.641509\n",
      "2    0.465116\n",
      "3    0.240000\n",
      "Name: is_correct, dtype: float64\n",
      "0.4878048780487805\n"
     ]
    }
   ],
   "source": [
    "third_run = result_df.loc[result_df[\"agent_name\"] == noanchorplan].copy()\n",
    "INCLUDE_THIRD_RUN = False\n",
    "\n",
    "\n",
    "# test ideal ensembling\n",
    "def score_best_both(row, result_df_replacement):\n",
    "    try:\n",
    "        if row[\"is_correct\"]:\n",
    "            return True\n",
    "\n",
    "        else:\n",
    "            matching_answer = result_df_replacement.loc[\n",
    "                (result_df_replacement[\"question\"] == row[\"question\"])\n",
    "            ].iloc[0]\n",
    "            if matching_answer[\"is_correct\"]:\n",
    "                return True\n",
    "            else:\n",
    "                return False\n",
    "    except:\n",
    "        return row[\"is_correct\"]\n",
    "\n",
    "\n",
    "combined_gpt4 = first_run_gpt4.copy()\n",
    "combined_gpt4[\"is_correct\"] = combined_gpt4.apply(\n",
    "    lambda x: score_best_both(x, second_run_gpt4), axis=1\n",
    ")\n",
    "if INCLUDE_THIRD_RUN:\n",
    "    combined_gpt4[\"is_correct\"] = combined_gpt4.apply(\n",
    "        lambda x: score_best_both(x, third_run), axis=1\n",
    "    )\n",
    "print(\"Ideal combined run:\")\n",
    "print(combined_gpt4.groupby([\"task\"])[\"is_correct\"].mean())\n",
    "print(combined_gpt4[\"is_correct\"].mean())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gaia",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}