MATHWELL8b_parse.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b75f9b59-51b6-4b6b-b3f4-289e5c2ad07d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...</td>\n",
       "      <td>def solution():\\n    #LeBron James is 6'9\" tal...</td>\n",
       "      <td>205.74</td>\n",
       "      <td>basketball</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Captain America has 1200 ryo. He spends 400 ry...</td>\n",
       "      <td>def solution():\\n    #Captain America started ...</td>\n",
       "      <td>300.0</td>\n",
       "      <td>Captain America</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3 scoops of ice cream cost $4. \\nHow much does...</td>\n",
       "      <td>def solution():\\n    #3 scoops of ice cream co...</td>\n",
       "      <td>13.333333333333332</td>\n",
       "      <td>ice cream</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>A volleyball team is 6 players. \\n3 of the pla...</td>\n",
       "      <td>def solution():\\n    #3 of the players are 16 ...</td>\n",
       "      <td>16.666666666666668</td>\n",
       "      <td>volleyball</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>Superman can fly at a speed of 1,000 miles per...</td>\n",
       "      <td>def solution():\\n    #Superman can fly at a sp...</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Superman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2088</th>\n",
       "      <td>1901</td>\n",
       "      <td>30 unicorns are in a herd. 10 unicorns in the ...</td>\n",
       "      <td>def solution():\\n    #Total number of unicorns...</td>\n",
       "      <td>17</td>\n",
       "      <td>unicorns</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2089</th>\n",
       "      <td>1902</td>\n",
       "      <td>A pizza has 8 slices. 4 of the slices have pep...</td>\n",
       "      <td>def solution():\\n    #A pizza has 8 slices\\n  ...</td>\n",
       "      <td>4</td>\n",
       "      <td>pizza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2090</th>\n",
       "      <td>1903</td>\n",
       "      <td>12000 people were at a rally. 10000 of them we...</td>\n",
       "      <td>def solution():\\n    #Total number of people a...</td>\n",
       "      <td>500</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2091</th>\n",
       "      <td>1904</td>\n",
       "      <td>A bakery made 300 muffins. 150 of the muffins ...</td>\n",
       "      <td>def solution():\\n    #Total muffins\\n    total...</td>\n",
       "      <td>60</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2092</th>\n",
       "      <td>1905</td>\n",
       "      <td>A pizza is cut into 12 slices. 5 slices are to...</td>\n",
       "      <td>def solution():\\n    #A pizza is cut into 12 s...</td>\n",
       "      <td>6</td>\n",
       "      <td>pizza</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2093 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                                           question  \\\n",
       "0              0  LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...   \n",
       "1              1  Captain America has 1200 ryo. He spends 400 ry...   \n",
       "2              3  3 scoops of ice cream cost $4. \\nHow much does...   \n",
       "3              5  A volleyball team is 6 players. \\n3 of the pla...   \n",
       "4              6  Superman can fly at a speed of 1,000 miles per...   \n",
       "...          ...                                                ...   \n",
       "2088        1901  30 unicorns are in a herd. 10 unicorns in the ...   \n",
       "2089        1902  A pizza has 8 slices. 4 of the slices have pep...   \n",
       "2090        1903  12000 people were at a rally. 10000 of them we...   \n",
       "2091        1904  A bakery made 300 muffins. 150 of the muffins ...   \n",
       "2092        1905  A pizza is cut into 12 slices. 5 slices are to...   \n",
       "\n",
       "                                               solution              answer  \\\n",
       "0     def solution():\\n    #LeBron James is 6'9\" tal...              205.74   \n",
       "1     def solution():\\n    #Captain America started ...               300.0   \n",
       "2     def solution():\\n    #3 scoops of ice cream co...  13.333333333333332   \n",
       "3     def solution():\\n    #3 of the players are 16 ...  16.666666666666668   \n",
       "4     def solution():\\n    #Superman can fly at a sp...                 0.2   \n",
       "...                                                 ...                 ...   \n",
       "2088  def solution():\\n    #Total number of unicorns...                  17   \n",
       "2089  def solution():\\n    #A pizza has 8 slices\\n  ...                   4   \n",
       "2090  def solution():\\n    #Total number of people a...                 500   \n",
       "2091  def solution():\\n    #Total muffins\\n    total...                  60   \n",
       "2092  def solution():\\n    #A pizza is cut into 12 s...                   6   \n",
       "\n",
       "                topic  \n",
       "0          basketball  \n",
       "1     Captain America  \n",
       "2           ice cream  \n",
       "3          volleyball  \n",
       "4            Superman  \n",
       "...               ...  \n",
       "2088         unicorns  \n",
       "2089            pizza  \n",
       "2090      unspecified  \n",
       "2091      unspecified  \n",
       "2092            pizza  \n",
       "\n",
       "[2093 rows x 5 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.stats as st\n",
    "finetune = pd.read_csv('data/sgsm_train.csv')\n",
    "finetune"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56344527-362a-416e-addc-4a55640b3659",
   "metadata": {},
   "source": [
    "## Llama 3 Zero Shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d0310b9c-0b58-43fe-ab8b-ef355d302b61",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150\n",
      "21.0\n",
      "129.0\n",
      "26.0\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "How many Grand Slam titles did Serena Williams win in 2017? 2\n",
      "How many Grand Slam titles did Serena Williams win in 2017? 2\n",
      "Enter Serena's weight in pounds:  2\n",
      "Enter the weight of a serving of tuna fish in grams:  2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "91.30434782608695% of Serena's titles are singles\n",
      "quarterback is not quarterback\n",
      "rate =  5\n",
      "26 Hello Kitties\n",
      "sword\n",
      "It will take Train 1 0 minute(s) to overtake Train 2.\n",
      "28\n",
      "1.5\n",
      "It will take Steve 12.5 digs to make a whole of blocks\n",
      "['Harry', 'Neville', 'Draco']\n",
      "['Harry', 'Neville', 'Draco']\n",
      "The player with the most cards is Neville\n",
      "\n",
      "I am Wonder Woman\n",
      "Wonder Woman is wearing a Wonder Woman outfit\n",
      "They Wonder Woman is wearing boots\n",
      "The Wonder Woman completed the Day\n",
      "The Wonder Woman decided to to put on her Wonder Woman boots again\n",
      "The Wonder Woman goes into the city to save the world\n",
      "12th\n",
      "12th\n",
      "11th\n",
      "11th\n",
      "4\n",
      "Steve has  4 irons left\n",
      "There are 262 cats in total.\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Please enter a number between 0-200: 22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n",
      "4.0\n",
      "80.0\n",
      "Chapter 1: 150 chapter_points\n",
      "Harry points have made: 150\n",
      "Chapter 2: 150 chapter_points\n",
      "Harry points have made: 300\n",
      "Harry points have made: 300\n",
      "Harry points have made: 300\n",
      "Chapter 5: 150 chapter_points\n",
      "Harry points have made: 450\n",
      "Percentage of Questions with Executable Code: 29.080000000000002\n",
      "Percentage of Questions with Executable Code (excluding non generations): 43.48086124401914\n",
      "1454 5000\n"
     ]
    }
   ],
   "source": [
    "file_path = 'llama3_8b_questions.txt'\n",
    "data = []\n",
    "\n",
    "with open(file_path, 'r') as file:\n",
    "    text = file.read()\n",
    "    sections = text.split(\"Topic: \")\n",
    "    #sections = text.split(\"### Response:\")\n",
    "    del sections[0]  # Remove the initial part before the question\n",
    "    total_questions = len(sections)\n",
    "    non_zero = total_questions\n",
    "    for section in sections:\n",
    "        if \"Question\" not in section:\n",
    "            non_zero-=1\n",
    "        try:\n",
    "            topic = section.split(\"### Response:\")[0].strip()\n",
    "            section = section.split(\"### Response:\")[1].strip()\n",
    "            question = section.split(\"Question:\")[1].strip()\n",
    "            question = question.split(\"Solution:\")[0].strip()\n",
    "            solution = section.split(\"Solution:\")[1].strip()\n",
    "            if 'while' in solution:\n",
    "                pass\n",
    "            if \"##\" in question:\n",
    "                question = question.split(\"##\")[0]\n",
    "#             if \"\\n\\n\" in question:\n",
    "#                 question = question.split(\"\\n\\n\")\n",
    "#                 question = \"\\n\".join(question)\n",
    "            if \"\\nBel\" in solution:\n",
    "                solution = solution.split(\"\\nBel\")[0]\n",
    "            if \"##\" in solution:\n",
    "                solution = solution.split(\"##\")[0]\n",
    "            if \"return\" not in solution:\n",
    "                try: \n",
    "                    solution = solution + '\\n' + '    return result'\n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                            # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, 'topic': topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                                pass\n",
    "                except:\n",
    "                    pass\n",
    "            if \"return\" in solution:\n",
    "                try: \n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                        # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                            pass\n",
    "                except:\n",
    "                    pass\n",
    "        except:\n",
    "            pass\n",
    "df1 = pd.DataFrame.from_dict(data)\n",
    "result1 = \"Percentage of Questions with Executable Code: \" + str((len(df1)/total_questions)*100)\n",
    "result2 = \"Percentage of Questions with Executable Code (excluding non generations): \" + str((len(df1)/non_zero)*100)\n",
    "print(result1)\n",
    "print(result2)\n",
    "print(len(df1), total_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4987a15b-a50e-4ea2-bb6a-1969bcd96236",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of questions that have 0 as answer: 4.607977991746905\n"
     ]
    }
   ],
   "source": [
    "print(f\"Percentage of questions that have 0 as answer: {(len(df1[df1['answer']==0])/len(df1))*100}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "4002fe40-c044-4eae-9160-764c395b7758",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df1.drop_duplicates(subset='question')\n",
    "df1 = df1.dropna(subset = 'topic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "1788de26-8b03-4026-b089-beab4121f886",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Captain America is preparing his shield for ba...</td>\n",
       "      <td>def solution():\\n    #Captain America throws h...</td>\n",
       "      <td>8.33</td>\n",
       "      <td>Captain America</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Captain Marvel flew a distance of 50 meters an...</td>\n",
       "      <td>def solution():\\n    #Captain Marvel flew 50 m...</td>\n",
       "      <td>85.0</td>\n",
       "      <td>planes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Captain Marvel has 500 shields. Her partner ha...</td>\n",
       "      <td>def solution():\\n    #Captain Marvel has 500 s...</td>\n",
       "      <td>900</td>\n",
       "      <td>Captain Marvel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Harry Potter lives at 4 Privet Drive.  There a...</td>\n",
       "      <td>def solution():\\n        houses_street = 11\\n ...</td>\n",
       "      <td>-17</td>\n",
       "      <td>Harry Potter</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Naruto and his companions will need 100 chakra...</td>\n",
       "      <td>def solution():\\n    #We know that Naruto and ...</td>\n",
       "      <td>10</td>\n",
       "      <td>Naruto</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1449</th>\n",
       "      <td>Serena is a 4 time Olympian. She took a bronze...</td>\n",
       "      <td>def solution():\\n    #Serena has won medals in...</td>\n",
       "      <td>7</td>\n",
       "      <td>Serena Williams</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1450</th>\n",
       "      <td>Green Ranger, the superhero, and Kat, the assi...</td>\n",
       "      <td>def solution():\\n    #Green Ranger and Kat hav...</td>\n",
       "      <td>280.0</td>\n",
       "      <td>Power Rangers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1451</th>\n",
       "      <td>There are 10 Power Rangers. 2 of them are from...</td>\n",
       "      <td>def solution():\\n    #total: 10 Power Rangers\\...</td>\n",
       "      <td>14</td>\n",
       "      <td>Captain America</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1452</th>\n",
       "      <td>There are 8 players on a soccer team. 7 of the...</td>\n",
       "      <td>def solution():\\n    #There are 8 players on a...</td>\n",
       "      <td>0</td>\n",
       "      <td>soccer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1453</th>\n",
       "      <td>The velociraptor ate 15 Triceratops heads one ...</td>\n",
       "      <td>def solution():\\n    #The velociraptor ate 15 ...</td>\n",
       "      <td>16.0</td>\n",
       "      <td>dinosaurs</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1439 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question  \\\n",
       "8     Captain America is preparing his shield for ba...   \n",
       "9     Captain Marvel flew a distance of 50 meters an...   \n",
       "10    Captain Marvel has 500 shields. Her partner ha...   \n",
       "11    Harry Potter lives at 4 Privet Drive.  There a...   \n",
       "12    Naruto and his companions will need 100 chakra...   \n",
       "...                                                 ...   \n",
       "1449  Serena is a 4 time Olympian. She took a bronze...   \n",
       "1450  Green Ranger, the superhero, and Kat, the assi...   \n",
       "1451  There are 10 Power Rangers. 2 of them are from...   \n",
       "1452  There are 8 players on a soccer team. 7 of the...   \n",
       "1453  The velociraptor ate 15 Triceratops heads one ...   \n",
       "\n",
       "                                               solution answer  \\\n",
       "8     def solution():\\n    #Captain America throws h...   8.33   \n",
       "9     def solution():\\n    #Captain Marvel flew 50 m...   85.0   \n",
       "10    def solution():\\n    #Captain Marvel has 500 s...    900   \n",
       "11    def solution():\\n        houses_street = 11\\n ...    -17   \n",
       "12    def solution():\\n    #We know that Naruto and ...     10   \n",
       "...                                                 ...    ...   \n",
       "1449  def solution():\\n    #Serena has won medals in...      7   \n",
       "1450  def solution():\\n    #Green Ranger and Kat hav...  280.0   \n",
       "1451  def solution():\\n    #total: 10 Power Rangers\\...     14   \n",
       "1452  def solution():\\n    #There are 8 players on a...      0   \n",
       "1453  def solution():\\n    #The velociraptor ate 15 ...   16.0   \n",
       "\n",
       "                topic  \n",
       "8     Captain America  \n",
       "9              planes  \n",
       "10     Captain Marvel  \n",
       "11       Harry Potter  \n",
       "12             Naruto  \n",
       "...               ...  \n",
       "1449  Serena Williams  \n",
       "1450    Power Rangers  \n",
       "1451  Captain America  \n",
       "1452           soccer  \n",
       "1453        dinosaurs  \n",
       "\n",
       "[1439 rows x 4 columns]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "03a71aa9-4040-48c5-8bcc-706899cf1c36",
   "metadata": {},
   "outputs": [],
   "source": [
    "question_numbers = []\n",
    "for i in range(0, len(df1)):\n",
    "    question_numbers.append(f\"llama3_q{i}\")\n",
    "df1['question_number'] = question_numbers\n",
    "df1_an = df1.sample(n=250, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "46977db9-40a5-4518-9937-4dd0e745eadb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1_an = df1_an[~df1_an['question'].isin(finetune['question'])]\n",
    "df1 = df1[~df1['question'].isin(finetune['question'])]\n",
    "df1 = df1[~df1['question'].isin(df1_an['question'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "0b743322-0af6-4fa7-8f2a-9286cb6daf7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "      <th>question_number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>LeBron James, an excellent basketball player, ...</td>\n",
       "      <td>def solution():\\n  #the challenge will never e...</td>\n",
       "      <td>The challenge will never end if it is left up ...</td>\n",
       "      <td>LeBron James</td>\n",
       "      <td>llama3_q168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>614</th>\n",
       "      <td>An airplane has 12 seats on each row. 72 seats...</td>\n",
       "      <td>def solution():\\n    #An airplane has 12 seats...</td>\n",
       "      <td>5</td>\n",
       "      <td>planes</td>\n",
       "      <td>llama3_q605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557</th>\n",
       "      <td>A basketball league has 12 teams in a division...</td>\n",
       "      <td>def solution():\\n    #Total number of teams\\n ...</td>\n",
       "      <td>300</td>\n",
       "      <td>basketball</td>\n",
       "      <td>llama3_q548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>Harry is about to battle Voldemort. If Voldemo...</td>\n",
       "      <td>def solution():\\n    #Voldemort has 4000 point...</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>Harry Potter</td>\n",
       "      <td>llama3_q65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637</th>\n",
       "      <td>There are 23 passenger cars on a train. 14 car...</td>\n",
       "      <td>def solution():\\n    #The train has 23 passeng...</td>\n",
       "      <td>9</td>\n",
       "      <td>trains</td>\n",
       "      <td>llama3_q628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>At an NBA game, LeBron James made 5 three-poin...</td>\n",
       "      <td>def solution():\\n    #LeBron James made 5 thre...</td>\n",
       "      <td>34</td>\n",
       "      <td>LeBron James</td>\n",
       "      <td>llama3_q196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1335</th>\n",
       "      <td>Superman ate 10% of the hamburgers at the burg...</td>\n",
       "      <td>def solution():\\n    #There are 100 hamburgers...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>Superman</td>\n",
       "      <td>llama3_q1321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>Darryl is running a candy machine at the airpo...</td>\n",
       "      <td>def solution():\\n    #Darryl starts with 200 g...</td>\n",
       "      <td>28</td>\n",
       "      <td>candy</td>\n",
       "      <td>llama3_q128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1220</th>\n",
       "      <td>You bring 30 cookies. 15 more cookies are brou...</td>\n",
       "      <td>def solution():\\n    #30 cookies brought by yo...</td>\n",
       "      <td>20</td>\n",
       "      <td>cookies</td>\n",
       "      <td>llama3_q1208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>The Boeing 737 has the most built so far of an...</td>\n",
       "      <td>def solution():\\n    #A Boeing 737 has the mos...</td>\n",
       "      <td>20000</td>\n",
       "      <td>planes</td>\n",
       "      <td>llama3_q81</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question  \\\n",
       "176   LeBron James, an excellent basketball player, ...   \n",
       "614   An airplane has 12 seats on each row. 72 seats...   \n",
       "557   A basketball league has 12 teams in a division...   \n",
       "73    Harry is about to battle Voldemort. If Voldemo...   \n",
       "637   There are 23 passenger cars on a train. 14 car...   \n",
       "...                                                 ...   \n",
       "204   At an NBA game, LeBron James made 5 three-poin...   \n",
       "1335  Superman ate 10% of the hamburgers at the burg...   \n",
       "136   Darryl is running a candy machine at the airpo...   \n",
       "1220  You bring 30 cookies. 15 more cookies are brou...   \n",
       "89    The Boeing 737 has the most built so far of an...   \n",
       "\n",
       "                                               solution  \\\n",
       "176   def solution():\\n  #the challenge will never e...   \n",
       "614   def solution():\\n    #An airplane has 12 seats...   \n",
       "557   def solution():\\n    #Total number of teams\\n ...   \n",
       "73    def solution():\\n    #Voldemort has 4000 point...   \n",
       "637   def solution():\\n    #The train has 23 passeng...   \n",
       "...                                                 ...   \n",
       "204   def solution():\\n    #LeBron James made 5 thre...   \n",
       "1335  def solution():\\n    #There are 100 hamburgers...   \n",
       "136   def solution():\\n    #Darryl starts with 200 g...   \n",
       "1220  def solution():\\n    #30 cookies brought by yo...   \n",
       "89    def solution():\\n    #A Boeing 737 has the mos...   \n",
       "\n",
       "                                                 answer         topic  \\\n",
       "176   The challenge will never end if it is left up ...  LeBron James   \n",
       "614                                                   5        planes   \n",
       "557                                                 300    basketball   \n",
       "73                                               2000.0  Harry Potter   \n",
       "637                                                   9        trains   \n",
       "...                                                 ...           ...   \n",
       "204                                                  34  LeBron James   \n",
       "1335                                               10.0      Superman   \n",
       "136                                                  28         candy   \n",
       "1220                                                 20       cookies   \n",
       "89                                                20000        planes   \n",
       "\n",
       "     question_number  \n",
       "176      llama3_q168  \n",
       "614      llama3_q605  \n",
       "557      llama3_q548  \n",
       "73        llama3_q65  \n",
       "637      llama3_q628  \n",
       "...              ...  \n",
       "204      llama3_q196  \n",
       "1335    llama3_q1321  \n",
       "136      llama3_q128  \n",
       "1220    llama3_q1208  \n",
       "89        llama3_q81  \n",
       "\n",
       "[250 rows x 5 columns]"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1_an"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7707d12c-ce38-4f43-b7a6-55f924c6c837",
   "metadata": {},
   "source": [
    "## MATHWELL8b Full Finetune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "f918bee1-0f4c-45b5-8393-979b28bcacf3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(37, 39)\n",
      "Percentage of Questions with Executable Code: 24.15\n",
      "Percentage of Questions with Executable Code (excluding non generations): 88.78676470588235\n",
      "483 2000\n"
     ]
    }
   ],
   "source": [
    "file_path = 'mathwell8b_full_questions.txt'\n",
    "data = []\n",
    "\n",
    "with open(file_path, 'r') as file:\n",
    "    text = file.read()\n",
    "    sections = text.split(\"Topic: \")\n",
    "    #sections = text.split(\"### Response:\")\n",
    "    del sections[0]  # Remove the initial part before the question\n",
    "    total_questions = len(sections)\n",
    "    non_zero = total_questions\n",
    "    for section in sections:\n",
    "        if \"Question\" not in section:\n",
    "            non_zero-=1\n",
    "        try:\n",
    "            topic = section.split(\"### Response:\")[0].strip()\n",
    "            section = section.split(\"### Response:\")[1].strip()\n",
    "            question = section.split(\"Question:\")[1].strip()\n",
    "            question = question.split(\"Solution:\")[0].strip()\n",
    "            solution = section.split(\"Solution:\")[1].strip()\n",
    "            if 'while' in solution:\n",
    "                pass\n",
    "            if \"##\" in question:\n",
    "                question = question.split(\"##\")[0]\n",
    "#             if \"\\n\\n\" in question:\n",
    "#                 question = question.split(\"\\n\\n\")\n",
    "#                 question = \"\\n\".join(question)\n",
    "            if \"\\nBel\" in solution:\n",
    "                solution = solution.split(\"\\nBel\")[0]\n",
    "            if \"##\" in solution:\n",
    "                solution = solution.split(\"##\")[0]\n",
    "            if \"return\" not in solution:\n",
    "                try: \n",
    "                    solution = solution + '\\n' + '    return result'\n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                            # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, 'topic': topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                                pass\n",
    "                except:\n",
    "                    pass\n",
    "            if \"return\" in solution:\n",
    "                try: \n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                        # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                            pass\n",
    "                except:\n",
    "                    pass\n",
    "        except:\n",
    "            pass\n",
    "df2 = pd.DataFrame.from_dict(data)\n",
    "result1 = \"Percentage of Questions with Executable Code: \" + str((len(df2)/total_questions)*100)\n",
    "result2 = \"Percentage of Questions with Executable Code (excluding non generations): \" + str((len(df2)/non_zero)*100)\n",
    "print(result1)\n",
    "print(result2)\n",
    "print(len(df2), total_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e15e4261-6342-4275-99b3-587b720959d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of questions that have 0 as answer: 4.968944099378882\n"
     ]
    }
   ],
   "source": [
    "print(f\"Percentage of questions that have 0 as answer: {(len(df2[df2['answer']==0])/len(df2))*100}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "4c6ad3b2-02d3-44db-adbe-363b2b3854c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "      <th>question_number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>A pilot is flying a plane that weighs 1000 pou...</td>\n",
       "      <td>def solution():\\n    #The plane weighs 1000 po...</td>\n",
       "      <td>6000</td>\n",
       "      <td>planes</td>\n",
       "      <td>mathwell8b_full_q93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321</th>\n",
       "      <td>Jules owns 25 cars. 10 of them are sports cars...</td>\n",
       "      <td>def solution():\\n    #Total cars\\n    total_ca...</td>\n",
       "      <td>13</td>\n",
       "      <td>cars</td>\n",
       "      <td>mathwell8b_full_q320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>20 unicorns were in a meadow. 5 of the unicorn...</td>\n",
       "      <td>def solution():\\n    #20 unicorns were in a me...</td>\n",
       "      <td>18</td>\n",
       "      <td>unicorns</td>\n",
       "      <td>mathwell8b_full_q55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>A unicorn ate 30 dandelions and 12 blueberries...</td>\n",
       "      <td>def solution():\\n    #The unicorn ate 30 dande...</td>\n",
       "      <td>360</td>\n",
       "      <td>unicorns</td>\n",
       "      <td>mathwell8b_full_q30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>1000000000 golden apples fell from the heavens...</td>\n",
       "      <td>def solution():\\n    #The total number of gold...</td>\n",
       "      <td>-1010000000</td>\n",
       "      <td>Wonder Woman</td>\n",
       "      <td>mathwell8b_full_q449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>Black Panther has 52 vibranium bullets. He fir...</td>\n",
       "      <td>def solution():\\n    #Black Panther has 52 vib...</td>\n",
       "      <td>22</td>\n",
       "      <td>Black Panther</td>\n",
       "      <td>mathwell8b_full_q96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>20 players on each team are on the field. 80% ...</td>\n",
       "      <td>def solution():\\n    #20 players on each team ...</td>\n",
       "      <td>3.2</td>\n",
       "      <td>lacrosse</td>\n",
       "      <td>mathwell8b_full_q143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>200 scoops of ice cream are on the conveyor be...</td>\n",
       "      <td>def solution():\\n    #Total scoops of ice crea...</td>\n",
       "      <td>150</td>\n",
       "      <td>ice cream</td>\n",
       "      <td>mathwell8b_full_q239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>10 friends are ordering pizza. Each of them wa...</td>\n",
       "      <td>def solution():\\n    #Number of friends\\n    f...</td>\n",
       "      <td>1.666667</td>\n",
       "      <td>pizza</td>\n",
       "      <td>mathwell8b_full_q434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>372</th>\n",
       "      <td>A parking lot has 200 spots. 120 of the spots ...</td>\n",
       "      <td>def solution():\\n    #Total spots\\n    total_s...</td>\n",
       "      <td>-80</td>\n",
       "      <td>cars</td>\n",
       "      <td>mathwell8b_full_q370</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              question  \\\n",
       "93   A pilot is flying a plane that weighs 1000 pou...   \n",
       "321  Jules owns 25 cars. 10 of them are sports cars...   \n",
       "55   20 unicorns were in a meadow. 5 of the unicorn...   \n",
       "30   A unicorn ate 30 dandelions and 12 blueberries...   \n",
       "453  1000000000 golden apples fell from the heavens...   \n",
       "..                                                 ...   \n",
       "96   Black Panther has 52 vibranium bullets. He fir...   \n",
       "143  20 players on each team are on the field. 80% ...   \n",
       "239  200 scoops of ice cream are on the conveyor be...   \n",
       "437  10 friends are ordering pizza. Each of them wa...   \n",
       "372  A parking lot has 200 spots. 120 of the spots ...   \n",
       "\n",
       "                                              solution       answer  \\\n",
       "93   def solution():\\n    #The plane weighs 1000 po...         6000   \n",
       "321  def solution():\\n    #Total cars\\n    total_ca...           13   \n",
       "55   def solution():\\n    #20 unicorns were in a me...           18   \n",
       "30   def solution():\\n    #The unicorn ate 30 dande...          360   \n",
       "453  def solution():\\n    #The total number of gold...  -1010000000   \n",
       "..                                                 ...          ...   \n",
       "96   def solution():\\n    #Black Panther has 52 vib...           22   \n",
       "143  def solution():\\n    #20 players on each team ...          3.2   \n",
       "239  def solution():\\n    #Total scoops of ice crea...          150   \n",
       "437  def solution():\\n    #Number of friends\\n    f...     1.666667   \n",
       "372  def solution():\\n    #Total spots\\n    total_s...          -80   \n",
       "\n",
       "             topic       question_number  \n",
       "93          planes   mathwell8b_full_q93  \n",
       "321           cars  mathwell8b_full_q320  \n",
       "55        unicorns   mathwell8b_full_q55  \n",
       "30        unicorns   mathwell8b_full_q30  \n",
       "453   Wonder Woman  mathwell8b_full_q449  \n",
       "..             ...                   ...  \n",
       "96   Black Panther   mathwell8b_full_q96  \n",
       "143       lacrosse  mathwell8b_full_q143  \n",
       "239      ice cream  mathwell8b_full_q239  \n",
       "437          pizza  mathwell8b_full_q434  \n",
       "372           cars  mathwell8b_full_q370  \n",
       "\n",
       "[250 rows x 5 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = df2.drop_duplicates(subset='question')\n",
    "df2 = df2[~df2['question'].isin(finetune['question'])]\n",
    "question_numbers = []\n",
    "for i in range(0, len(df2)):\n",
    "    question_numbers.append(f\"mathwell8b_full_q{i}\")\n",
    "df2['question_number'] = question_numbers\n",
    "df2_an = df2.sample(n=min(250,len(df2)), random_state = 42)\n",
    "df2_an"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e913fcc1-8acf-4369-ab1b-b0a92bdf1e2a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "      <th>question_number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A player has 12000 experience points. Experien...</td>\n",
       "      <td>def solution():\\n    #The player has 12000 exp...</td>\n",
       "      <td>120.0</td>\n",
       "      <td>Minecraft</td>\n",
       "      <td>mathwell8b_full_q0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hello Kitty has 500 pencils. 200 of the pencil...</td>\n",
       "      <td>def solution():\\n    #Hello Kitty has 500 penc...</td>\n",
       "      <td>150</td>\n",
       "      <td>Hello Kitty</td>\n",
       "      <td>mathwell8b_full_q1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Serena Williams has 200 trophies. 100 of them ...</td>\n",
       "      <td>def solution():\\n    #Total trophies\\n    tota...</td>\n",
       "      <td>100</td>\n",
       "      <td>Serena Williams</td>\n",
       "      <td>mathwell8b_full_q2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10,000 dogs are barking. 1/4 of the dogs are R...</td>\n",
       "      <td>def solution():\\n    #Total dogs\\n    total_do...</td>\n",
       "      <td>500.0</td>\n",
       "      <td>dogs</td>\n",
       "      <td>mathwell8b_full_q3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>300 Barbie dolls were in a toy store. 120 of t...</td>\n",
       "      <td>def solution():\\n    #300 Barbie dolls were in...</td>\n",
       "      <td>120</td>\n",
       "      <td>Barbie</td>\n",
       "      <td>mathwell8b_full_q4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478</th>\n",
       "      <td>Serena Williams has 200 Wimbledon titles. 100 ...</td>\n",
       "      <td>def solution():\\n    #Serena Williams has 200 ...</td>\n",
       "      <td>50</td>\n",
       "      <td>Serena Williams</td>\n",
       "      <td>mathwell8b_full_q474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>In volleyball, each team has 6 players on the ...</td>\n",
       "      <td>def solution():\\n    #Each team has 6 players\\...</td>\n",
       "      <td>49.333333</td>\n",
       "      <td>volleyball</td>\n",
       "      <td>mathwell8b_full_q475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480</th>\n",
       "      <td>A 12-person soccer team has 32 shirts. \\n8 of ...</td>\n",
       "      <td>def solution():\\n    #A 12-person soccer team ...</td>\n",
       "      <td>18</td>\n",
       "      <td>football</td>\n",
       "      <td>mathwell8b_full_q476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>481</th>\n",
       "      <td>How many scoops of ice cream can you get from ...</td>\n",
       "      <td>def solution():\\n    #1 scoop of ice cream is ...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>ice cream</td>\n",
       "      <td>mathwell8b_full_q477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482</th>\n",
       "      <td>11 players on the field. 22 players on the ben...</td>\n",
       "      <td>def solution():\\n    #11 players on the field\\...</td>\n",
       "      <td>50.0</td>\n",
       "      <td>football</td>\n",
       "      <td>mathwell8b_full_q478</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>479 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              question  \\\n",
       "0    A player has 12000 experience points. Experien...   \n",
       "1    Hello Kitty has 500 pencils. 200 of the pencil...   \n",
       "2    Serena Williams has 200 trophies. 100 of them ...   \n",
       "3    10,000 dogs are barking. 1/4 of the dogs are R...   \n",
       "4    300 Barbie dolls were in a toy store. 120 of t...   \n",
       "..                                                 ...   \n",
       "478  Serena Williams has 200 Wimbledon titles. 100 ...   \n",
       "479  In volleyball, each team has 6 players on the ...   \n",
       "480  A 12-person soccer team has 32 shirts. \\n8 of ...   \n",
       "481  How many scoops of ice cream can you get from ...   \n",
       "482  11 players on the field. 22 players on the ben...   \n",
       "\n",
       "                                              solution     answer  \\\n",
       "0    def solution():\\n    #The player has 12000 exp...      120.0   \n",
       "1    def solution():\\n    #Hello Kitty has 500 penc...        150   \n",
       "2    def solution():\\n    #Total trophies\\n    tota...        100   \n",
       "3    def solution():\\n    #Total dogs\\n    total_do...      500.0   \n",
       "4    def solution():\\n    #300 Barbie dolls were in...        120   \n",
       "..                                                 ...        ...   \n",
       "478  def solution():\\n    #Serena Williams has 200 ...         50   \n",
       "479  def solution():\\n    #Each team has 6 players\\...  49.333333   \n",
       "480  def solution():\\n    #A 12-person soccer team ...         18   \n",
       "481  def solution():\\n    #1 scoop of ice cream is ...        4.0   \n",
       "482  def solution():\\n    #11 players on the field\\...       50.0   \n",
       "\n",
       "               topic       question_number  \n",
       "0          Minecraft    mathwell8b_full_q0  \n",
       "1        Hello Kitty    mathwell8b_full_q1  \n",
       "2    Serena Williams    mathwell8b_full_q2  \n",
       "3               dogs    mathwell8b_full_q3  \n",
       "4             Barbie    mathwell8b_full_q4  \n",
       "..               ...                   ...  \n",
       "478  Serena Williams  mathwell8b_full_q474  \n",
       "479       volleyball  mathwell8b_full_q475  \n",
       "480         football  mathwell8b_full_q476  \n",
       "481        ice cream  mathwell8b_full_q477  \n",
       "482         football  mathwell8b_full_q478  \n",
       "\n",
       "[479 rows x 5 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e304e11-c054-490c-8d0e-21781712a68f",
   "metadata": {},
   "source": [
    "## MATHWELL 8b Two Stage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "c5fb2f00-927c-4f38-b8c3-85db9456afec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of Questions with Executable Code: 20.45\n",
      "Percentage of Questions with Executable Code (excluding non generations): 87.20682302771856\n",
      "409 2000\n"
     ]
    }
   ],
   "source": [
    "file_path = 'mathwell8b_two_stage_questions.txt'\n",
    "data = []\n",
    "\n",
    "with open(file_path, 'r') as file:\n",
    "    text = file.read()\n",
    "    sections = text.split(\"Topic: \")\n",
    "    #sections = text.split(\"### Response:\")\n",
    "    del sections[0]  # Remove the initial part before the question\n",
    "    total_questions = len(sections)\n",
    "    non_zero = total_questions\n",
    "    for section in sections:\n",
    "        if \"Question\" not in section:\n",
    "            non_zero-=1\n",
    "        try:\n",
    "            topic = section.split(\"### Response:\")[0].strip()\n",
    "            section = section.split(\"### Response:\")[1].strip()\n",
    "            question = section.split(\"Question:\")[1].strip()\n",
    "            question = question.split(\"Solution:\")[0].strip()\n",
    "            solution = section.split(\"Solution:\")[1].strip()\n",
    "            if 'while' in solution:\n",
    "                pass\n",
    "            if \"##\" in question:\n",
    "                question = question.split(\"##\")[0]\n",
    "#             if \"\\n\\n\" in question:\n",
    "#                 question = question.split(\"\\n\\n\")\n",
    "#                 question = \"\\n\".join(question)\n",
    "            if \"\\nBel\" in solution:\n",
    "                solution = solution.split(\"\\nBel\")[0]\n",
    "            if \"##\" in solution:\n",
    "                solution = solution.split(\"##\")[0]\n",
    "            if \"return\" not in solution:\n",
    "                try: \n",
    "                    solution = solution + '\\n' + '    return result'\n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                            # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, 'topic': topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                                pass\n",
    "                except:\n",
    "                    pass\n",
    "            if \"return\" in solution:\n",
    "                try: \n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                        # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                            pass\n",
    "                except:\n",
    "                    pass\n",
    "        except:\n",
    "            pass\n",
    "df3 = pd.DataFrame.from_dict(data)\n",
    "result1 = \"Percentage of Questions with Executable Code: \" + str((len(df3)/total_questions)*100)\n",
    "result2 = \"Percentage of Questions with Executable Code (excluding non generations): \" + str((len(df3)/non_zero)*100)\n",
    "print(result1)\n",
    "print(result2)\n",
    "print(len(df3), total_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "44d77d72-af72-444d-ac8d-05ea29c2683f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of questions that have 0 as answer: 5.623471882640587\n"
     ]
    }
   ],
   "source": [
    "print(f\"Percentage of questions that have 0 as answer: {(len(df3[df3['answer']==0])/len(df3))*100}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "facddddb-ce46-4b97-9c41-a6b87c7df5cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = df3.drop_duplicates(subset='question')\n",
    "df3 = df3[~df3['question'].isin(finetune['question'])]\n",
    "question_numbers = []\n",
    "for i in range(0, len(df3)):\n",
    "    question_numbers.append(f\"mathwell8b_two_stage_q{i}\")\n",
    "df3['question_number'] = question_numbers\n",
    "df3_an = df3.sample(n=min(250,len(df3)), random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "a1aaac67-699b-4c0c-8734-124da20891a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "      <th>question_number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A Unicorn breeder is collecting unicorns. He h...</td>\n",
       "      <td>def solution():\\n    #The Unicorn breeder star...</td>\n",
       "      <td>250</td>\n",
       "      <td>unicorns</td>\n",
       "      <td>mathwell8b_two_stage_q0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A scoop of vanilla ice cream is $3 and a scoop...</td>\n",
       "      <td>def solution():\\n    #A scoop of vanilla ice c...</td>\n",
       "      <td>3</td>\n",
       "      <td>ice cream</td>\n",
       "      <td>mathwell8b_two_stage_q1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Serena Williams is a 4-time Olympic gold medal...</td>\n",
       "      <td>def solution():\\n    #Serena has 3 gold medals...</td>\n",
       "      <td>0</td>\n",
       "      <td>Serena Williams</td>\n",
       "      <td>mathwell8b_two_stage_q2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LeBron James has 50000 points. He averages 30 ...</td>\n",
       "      <td>def solution():\\n    #LeBron James has 50000 p...</td>\n",
       "      <td>666.666667</td>\n",
       "      <td>LeBron James</td>\n",
       "      <td>mathwell8b_two_stage_q3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Barbie has 2000 dolls. She sells 300 of them i...</td>\n",
       "      <td>def solution():\\n    #Barbie started with 2000...</td>\n",
       "      <td>1000</td>\n",
       "      <td>Barbie</td>\n",
       "      <td>mathwell8b_two_stage_q4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404</th>\n",
       "      <td>Superman can fly 1000 miles per hour.  He is 3...</td>\n",
       "      <td>def solution():\\n    #Superman can fly 1000 mi...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Superman</td>\n",
       "      <td>mathwell8b_two_stage_q389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405</th>\n",
       "      <td>Superman is 1000 times stronger than a human. ...</td>\n",
       "      <td>def solution():\\n    #A human can lift 100 pou...</td>\n",
       "      <td>100000</td>\n",
       "      <td>Superman</td>\n",
       "      <td>mathwell8b_two_stage_q390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>406</th>\n",
       "      <td>1000 cars are in a parking lot. 200 of the car...</td>\n",
       "      <td>def solution():\\n    #1000 cars are in a parki...</td>\n",
       "      <td>400</td>\n",
       "      <td>cars</td>\n",
       "      <td>mathwell8b_two_stage_q391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>407</th>\n",
       "      <td>1000000000 V-Bucks is the in-game currency in ...</td>\n",
       "      <td>def solution():\\n    #1000000000 V-Bucks is 10...</td>\n",
       "      <td>900000000</td>\n",
       "      <td>Fortnite</td>\n",
       "      <td>mathwell8b_two_stage_q392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>408</th>\n",
       "      <td>A T-Rex is 30 feet long. \\nA Brachiosaurus is ...</td>\n",
       "      <td>def solution():\\n    #A T-Rex is 30 feet long\\...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>dinosaurs</td>\n",
       "      <td>mathwell8b_two_stage_q393</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>394 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              question  \\\n",
       "0    A Unicorn breeder is collecting unicorns. He h...   \n",
       "1    A scoop of vanilla ice cream is $3 and a scoop...   \n",
       "2    Serena Williams is a 4-time Olympic gold medal...   \n",
       "3    LeBron James has 50000 points. He averages 30 ...   \n",
       "4    Barbie has 2000 dolls. She sells 300 of them i...   \n",
       "..                                                 ...   \n",
       "404  Superman can fly 1000 miles per hour.  He is 3...   \n",
       "405  Superman is 1000 times stronger than a human. ...   \n",
       "406  1000 cars are in a parking lot. 200 of the car...   \n",
       "407  1000000000 V-Bucks is the in-game currency in ...   \n",
       "408  A T-Rex is 30 feet long. \\nA Brachiosaurus is ...   \n",
       "\n",
       "                                              solution      answer  \\\n",
       "0    def solution():\\n    #The Unicorn breeder star...         250   \n",
       "1    def solution():\\n    #A scoop of vanilla ice c...           3   \n",
       "2    def solution():\\n    #Serena has 3 gold medals...           0   \n",
       "3    def solution():\\n    #LeBron James has 50000 p...  666.666667   \n",
       "4    def solution():\\n    #Barbie started with 2000...        1000   \n",
       "..                                                 ...         ...   \n",
       "404  def solution():\\n    #Superman can fly 1000 mi...         3.0   \n",
       "405  def solution():\\n    #A human can lift 100 pou...      100000   \n",
       "406  def solution():\\n    #1000 cars are in a parki...         400   \n",
       "407  def solution():\\n    #1000000000 V-Bucks is 10...   900000000   \n",
       "408  def solution():\\n    #A T-Rex is 30 feet long\\...         5.0   \n",
       "\n",
       "               topic            question_number  \n",
       "0           unicorns    mathwell8b_two_stage_q0  \n",
       "1          ice cream    mathwell8b_two_stage_q1  \n",
       "2    Serena Williams    mathwell8b_two_stage_q2  \n",
       "3       LeBron James    mathwell8b_two_stage_q3  \n",
       "4             Barbie    mathwell8b_two_stage_q4  \n",
       "..               ...                        ...  \n",
       "404         Superman  mathwell8b_two_stage_q389  \n",
       "405         Superman  mathwell8b_two_stage_q390  \n",
       "406             cars  mathwell8b_two_stage_q391  \n",
       "407         Fortnite  mathwell8b_two_stage_q392  \n",
       "408        dinosaurs  mathwell8b_two_stage_q393  \n",
       "\n",
       "[394 rows x 5 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "52d5b8af-4490-4ea4-90a9-348377dc8117",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df3 = df3[~df3['question'].isin(df3_an['question'])]\n",
    "# extra_an2 = df2.sample(n=250-len(df2_an), random_state = 42)\n",
    "# extra_an2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "077aa003-d5c5-4457-8ed2-6cde0ab37598",
   "metadata": {},
   "source": [
    "## MATHWELL 8B KTO Few Shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8bad2a02-57fd-480a-be6f-3a9cc3b14613",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of Questions with Executable Code: 79.88898377455168\n",
      "Percentage of Questions with Executable Code (excluding non generations): 92.34945705824285\n",
      "1871 2342\n"
     ]
    }
   ],
   "source": [
    "file_path = 'mathwell8b_kto_questions_few.txt'\n",
    "data = []\n",
    "\n",
    "with open(file_path, 'r') as file:\n",
    "    text = file.read()\n",
    "    sections = text.split(\"Topic: \")\n",
    "    #sections = text.split(\"### Response:\")\n",
    "    del sections[0]  # Remove the initial part before the question\n",
    "    total_questions = len(sections)\n",
    "    non_zero = total_questions\n",
    "    for section in sections:\n",
    "        if \"Question\" not in section:\n",
    "            non_zero-=1\n",
    "        try:\n",
    "            topic = section.split(\"### Response:\")[0].strip()\n",
    "            section = section.split(\"### Response:\")[1].strip()\n",
    "            question = section.split(\"Question:\")[1].strip()\n",
    "            question = question.split(\"Solution:\")[0].strip()\n",
    "            solution = section.split(\"Solution:\")[1].strip()\n",
    "            if 'while' in solution:\n",
    "                pass\n",
    "            if \"##\" in question:\n",
    "                question = question.split(\"##\")[0]\n",
    "#             if \"\\n\\n\" in question:\n",
    "#                 question = question.split(\"\\n\\n\")\n",
    "#                 question = \"\\n\".join(question)\n",
    "            if \"\\nBel\" in solution:\n",
    "                solution = solution.split(\"\\nBel\")[0]\n",
    "            if \"##\" in solution:\n",
    "                solution = solution.split(\"##\")[0]\n",
    "            if \"return\" not in solution:\n",
    "                try: \n",
    "                    solution = solution + '\\n' + '    return result'\n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                            # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, 'topic': topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                                pass\n",
    "                except:\n",
    "                    pass\n",
    "            if \"return\" in solution:\n",
    "                try: \n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                        # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                            pass\n",
    "                except:\n",
    "                    pass\n",
    "        except:\n",
    "            pass\n",
    "df4 = pd.DataFrame.from_dict(data)\n",
    "result1 = \"Percentage of Questions with Executable Code: \" + str((len(df4)/total_questions)*100)\n",
    "result2 = \"Percentage of Questions with Executable Code (excluding non generations): \" + str((len(df4)/non_zero)*100)\n",
    "print(result1)\n",
    "print(result2)\n",
    "print(len(df4), total_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "168a4939-1468-44a4-b0f6-24da6f08c747",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of questions that have 0 as answer: 3.7947621592731156\n"
     ]
    }
   ],
   "source": [
    "print(f\"Percentage of questions that have 0 as answer: {(len(df4[df4['answer']==0])/len(df4))*100}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7230d8e6-1ba1-4997-93f6-cb330b61a6d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1772\n",
      "1655\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "      <th>question_number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Taylor Swift has 2000 fans. 1000 of them are i...</td>\n",
       "      <td>def solution():\\n    #Total fans\\n    total_fa...</td>\n",
       "      <td>500</td>\n",
       "      <td>Taylor Swift</td>\n",
       "      <td>mathwell8b_kto_q0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Super Mario has 300 coins. He spends 50 coins ...</td>\n",
       "      <td>def solution():\\n    #Super Mario started with...</td>\n",
       "      <td>200</td>\n",
       "      <td>Super Mario</td>\n",
       "      <td>mathwell8b_kto_q1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Thor has 2000 lightning bolts. He uses 1000 of...</td>\n",
       "      <td>def solution():\\n    #Thor started with 2000 l...</td>\n",
       "      <td>300</td>\n",
       "      <td>Thor, the God of Thunder</td>\n",
       "      <td>mathwell8b_kto_q2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Patrick Mahomes is the quarterback for the Kan...</td>\n",
       "      <td>def solution():\\n    #Patrick Mahomes has 3000...</td>\n",
       "      <td>2800</td>\n",
       "      <td>Patrick Mahomes</td>\n",
       "      <td>mathwell8b_kto_q3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Steph Curry is a basketball player. He made 12...</td>\n",
       "      <td>def solution():\\n    #Steph Curry made 12 thre...</td>\n",
       "      <td>120.0</td>\n",
       "      <td>Steph Curry</td>\n",
       "      <td>mathwell8b_kto_q4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1866</th>\n",
       "      <td>Steph Curry has 2000 3-pointers. He has 1000 m...</td>\n",
       "      <td>def solution():\\n    #Steph Curry has 2000 3-p...</td>\n",
       "      <td>17000</td>\n",
       "      <td>basketball</td>\n",
       "      <td>mathwell8b_kto_q1650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1867</th>\n",
       "      <td>Taylor Swift has 2000 fans. 1000 of them are i...</td>\n",
       "      <td>def solution():\\n    #Total fans\\n    fans_tot...</td>\n",
       "      <td>500</td>\n",
       "      <td>Taylor Swift</td>\n",
       "      <td>mathwell8b_kto_q1651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1868</th>\n",
       "      <td>Serena Williams has 18 Grand Slam titles. 13 o...</td>\n",
       "      <td>def solution():\\n    #Serena Williams has 18 G...</td>\n",
       "      <td>4</td>\n",
       "      <td>Serena Williams</td>\n",
       "      <td>mathwell8b_kto_q1652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1869</th>\n",
       "      <td>1200 cookies are in a jar. 200 of the cookies ...</td>\n",
       "      <td>def solution():\\n    #Total number of cookies\\...</td>\n",
       "      <td>0</td>\n",
       "      <td>cookies</td>\n",
       "      <td>mathwell8b_kto_q1653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1870</th>\n",
       "      <td>2000 cookies are baked by a baker. 500 of the ...</td>\n",
       "      <td>def solution():\\n    #2000 cookies are baked b...</td>\n",
       "      <td>700</td>\n",
       "      <td>cookies</td>\n",
       "      <td>mathwell8b_kto_q1654</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1655 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question  \\\n",
       "0     Taylor Swift has 2000 fans. 1000 of them are i...   \n",
       "1     Super Mario has 300 coins. He spends 50 coins ...   \n",
       "3     Thor has 2000 lightning bolts. He uses 1000 of...   \n",
       "4     Patrick Mahomes is the quarterback for the Kan...   \n",
       "5     Steph Curry is a basketball player. He made 12...   \n",
       "...                                                 ...   \n",
       "1866  Steph Curry has 2000 3-pointers. He has 1000 m...   \n",
       "1867  Taylor Swift has 2000 fans. 1000 of them are i...   \n",
       "1868  Serena Williams has 18 Grand Slam titles. 13 o...   \n",
       "1869  1200 cookies are in a jar. 200 of the cookies ...   \n",
       "1870  2000 cookies are baked by a baker. 500 of the ...   \n",
       "\n",
       "                                               solution answer  \\\n",
       "0     def solution():\\n    #Total fans\\n    total_fa...    500   \n",
       "1     def solution():\\n    #Super Mario started with...    200   \n",
       "3     def solution():\\n    #Thor started with 2000 l...    300   \n",
       "4     def solution():\\n    #Patrick Mahomes has 3000...   2800   \n",
       "5     def solution():\\n    #Steph Curry made 12 thre...  120.0   \n",
       "...                                                 ...    ...   \n",
       "1866  def solution():\\n    #Steph Curry has 2000 3-p...  17000   \n",
       "1867  def solution():\\n    #Total fans\\n    fans_tot...    500   \n",
       "1868  def solution():\\n    #Serena Williams has 18 G...      4   \n",
       "1869  def solution():\\n    #Total number of cookies\\...      0   \n",
       "1870  def solution():\\n    #2000 cookies are baked b...    700   \n",
       "\n",
       "                         topic       question_number  \n",
       "0                 Taylor Swift     mathwell8b_kto_q0  \n",
       "1                  Super Mario     mathwell8b_kto_q1  \n",
       "3     Thor, the God of Thunder     mathwell8b_kto_q2  \n",
       "4              Patrick Mahomes     mathwell8b_kto_q3  \n",
       "5                  Steph Curry     mathwell8b_kto_q4  \n",
       "...                        ...                   ...  \n",
       "1866                basketball  mathwell8b_kto_q1650  \n",
       "1867              Taylor Swift  mathwell8b_kto_q1651  \n",
       "1868           Serena Williams  mathwell8b_kto_q1652  \n",
       "1869                   cookies  mathwell8b_kto_q1653  \n",
       "1870                   cookies  mathwell8b_kto_q1654  \n",
       "\n",
       "[1655 rows x 5 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df4 = df4.drop_duplicates(subset='question')\n",
    "print(len(df4))\n",
    "df4 = df4[~df4['question'].isin(finetune['question'])]\n",
    "print(len(df4))\n",
    "question_numbers = []\n",
    "for i in range(0, len(df4)):\n",
    "    question_numbers.append(f\"mathwell8b_kto_q{i}\")\n",
    "df4['question_number'] = question_numbers\n",
    "df4_an = df4.sample(n=min(250,len(df4)), random_state = 42)\n",
    "df4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3fc965c-9a94-42c4-a6c8-671e684ac875",
   "metadata": {},
   "source": [
    "# Explore Random Samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cecdf39a-afd6-47b5-aefe-1eda6da22ee0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: Iron Man has 2000 gold coins. He spends 500 gold coins on a new suit of armor. He then spends 200 gold coins on a new AI. He spends 300 gold coins on a new repulsor beam. How many gold coins does he have left?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Iron Man started with 2000 gold coins\n",
      "    gold_coins_initial = 2000\n",
      "    #He spent 500 on a new suit of armor\n",
      "    gold_coins_suit = 500\n",
      "    #He spent 200 on a new AI\n",
      "    gold_coins_ai = 200\n",
      "    #He spent 300 on a new repulsor beam\n",
      "    gold_coins_repulsor = 300\n",
      "    #The answer is\n",
      "    result = gold_coins_initial - gold_coins_suit - gold_coins_ai - gold_coins_repulsor\n",
      "    return result\n",
      "Question: Steph Curry, the point guard for the Golden State Warriors, is the best 3-point shooter in the NBA. He has made 3-pointers in 24 consecutive games. In each of the first 12 games, he made 5 3-pointers. In each of the next 12 games, he made 6 3-pointers. How many 3-pointers did he make in the 24 games?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Steph Curry made 5 3-pointers in each of the first 12 games\n",
      "    three_pointers_first_12_games = 12 * 5\n",
      "    #He made 6 3-pointers in each of the next 12 games\n",
      "    three_pointers_next_12_games = 12 * 6\n",
      "    #The answer is\n",
      "    result = three_pointers_first_12_games + three_pointers_next_12_games\n",
      "    return result\n",
      "Question: Barbie has 100 pink shoes. She has 30 more pink shoes than black shoes. How many black shoes does she have?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Barbie has 100 pink shoes\n",
      "    pink_shoes = 100\n",
      "    #She has 30 more pink shoes than black shoes\n",
      "    black_shoes = pink_shoes - 30\n",
      "    #The answer is\n",
      "    result = black_shoes\n",
      "    return result\n",
      "Question: Patrick Mahomes is the starting quarterback for the Kansas City Chiefs. He has 1000 passing yards. He gains 250 passing yards in each of the next 3 games. How many passing yards does he have now?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Patrick Mahomes started with 1000 passing yards\n",
      "    passing_yards_initial = 1000\n",
      "    #He gained 250 passing yards in each of the next 3 games\n",
      "    passing_yards_gained_per_game = 250\n",
      "    #There are 3 games\n",
      "    num_games = 3\n",
      "    #The answer is\n",
      "    result = passing_yards_initial + passing_yards_gained_per_game * num_games\n",
      "    return result\n",
      "Question: Wonder Woman has 200 lassos.  She used 20 of them to capture the Cheetah.  She used 30 of them to capture the Silver Swan.  She used 50 of them to capture the Circe.  How many lassos does she have left?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Wonder Woman started with 200 lassos\n",
      "    lassos_initial = 200\n",
      "    #She used 20 to capture the Cheetah\n",
      "    lassos_used_cheetah = 20\n",
      "    #She used 30 to capture the Silver Swan\n",
      "    lassos_used_silver_swan = 30\n",
      "    #She used 50 to capture the Circe\n",
      "    lassos_used_circe = 50\n",
      "    #The answer is\n",
      "    result = lassos_initial - lassos_used_cheetah - lassos_used_silver_swan - lassos_used_circe\n",
      "    return result\n",
      "Question: If 1/2 of a 1/2 gallon of ice cream is 120 ml, how many ml is 1/2 gallon of ice cream?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #1/2 of a 1/2 gallon of ice cream is 120 ml\n",
      "    half_gallon_half = 120\n",
      "    #1 gallon of ice cream is 2 * 1/2 gallon of ice cream\n",
      "    gallon = 2 * half_gallon_half\n",
      "    #The answer is\n",
      "    result = gallon\n",
      "    return result\n",
      "Question: LeBron James is 6'9\" tall.  He can reach 13'9\" high to tap an opponent's ball.  How many inches higher is he to reach to tap the ball than he is tall?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #LeBron's height in inches\n",
      "    lebron_height = 6 * 12 + 9\n",
      "    #He can reach 13'9\" high\n",
      "    lebron_reach = 13 * 12 + 9\n",
      "    #The answer is\n",
      "    result = lebron_reach - lebron_height\n",
      "    return result\n",
      "Question: 50 cats are in a cat park. 10 more cats show up. 5 of the cats that are in the park leave. How many cats are now in the park?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #50 cats are in a cat park\n",
      "    cats_initial = 50\n",
      "    #10 more cats show up\n",
      "    cats_showed_up = 10\n",
      "    #5 of the cats that are in the park leave\n",
      "    cats_left = 5\n",
      "    #The answer is\n",
      "    result = cats_initial + cats_showed_up - cats_left\n",
      "    return result\n",
      "Question: Serena Williams has 14 Grand Slam singles titles. She has 3 more than twice as many Grand Slam doubles titles as singles titles. How many Grand Slam doubles titles does she have?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Serena Williams has 14 Grand Slam singles titles\n",
      "    singles_titles = 14\n",
      "    #She has 3 more than twice as many Grand Slam doubles titles as singles titles\n",
      "    doubles_titles = 2 * singles_titles + 3\n",
      "    #The answer is\n",
      "    result = doubles_titles\n",
      "    return result\n",
      "Question: A cheerleading squad has 20 members. 8 of the members are in 10th grade. 6 of the members are in 11th grade. 4 of the members are in 12th grade. How many members are in 10th or 11th grade?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #A cheerleading squad has 20 members\n",
      "    total_members = 20\n",
      "    #8 of the members are in 10th grade\n",
      "    tenth_grade_members = 8\n",
      "    #6 of the members are in 11th grade\n",
      "    eleventh_grade_members = 6\n",
      "    #4 of the members are in 12th grade\n",
      "    twelfth_grade_members = 4\n",
      "    #The answer is\n",
      "    result = tenth_grade_members + eleventh_grade_members\n",
      "    return result\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "for i in range(0, 10):\n",
    "    index = random.randint(0, len(df4))\n",
    "    question = df4.iloc[index]['question']\n",
    "    solution = df4.iloc[index]['solution']\n",
    "    print(f\"Question: {question}\\n\\nSolution:\\n{solution}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "879ef919-2653-447d-b4b6-e47282123a7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: A football team has 100 players. \n",
      "\n",
      "40 of the players are on the offense. \n",
      "\n",
      "35 of the players are on the defense. \n",
      "\n",
      "The rest of the players are on special teams. \n",
      "\n",
      "How many players are on special teams?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #A football team has 100 players\n",
      "    players_total = 100\n",
      "    #40 of the players are on the offense\n",
      "    players_offense = 40\n",
      "    #35 of the players are on the defense\n",
      "    players_defense = 35\n",
      "    #The answer is\n",
      "    players_special_teams = players_total - players_offense - players_defense\n",
      "    return players_special_teams\n",
      "Question: Serena Williams has 200 Wimbledon titles. 100 of them are in singles, 50 are in doubles, and the rest are in mixed doubles. How many Wimbledon titles does she have in mixed doubles?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Serena Williams has 200 Wimbledon titles\n",
      "    total_titles = 200\n",
      "    #100 of them are in singles\n",
      "    singles_titles = 100\n",
      "    #50 are in doubles\n",
      "    doubles_titles = 50\n",
      "    #The rest are in mixed doubles\n",
      "    mixed_doubles_titles = total_titles - singles_titles - doubles_titles\n",
      "    return mixed_doubles_titles\n",
      "Question: Patrick Mahomes has 200 passing yards. He passes for 2 touchdowns and 1 interception. How many net yards does he have?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Patrick Mahomes started with 200 passing yards\n",
      "    passing_yards_initial = 200\n",
      "    #He passed for 2 touchdowns\n",
      "    touchdowns = 2\n",
      "    #He had 1 interception\n",
      "    interceptions = 1\n",
      "    #A touchdown is worth 10 yards, and an interception is worth -10 yards\n",
      "    touchdown_yards = touchdowns * 10\n",
      "    interception_yards = interceptions * -10\n",
      "    #The total net yards is\n",
      "    result = passing_yards_initial + touchdown_yards + interception_yards\n",
      "    return result\n",
      "Question: Red, Blue, and Yellow Power Ranger each have 8 Power Gems. Pink and Green Power Ranger each have 2 Power Gems. How many more Power Gems do Red, Blue, and Yellow Power Ranger have than Pink and Green Power Ranger?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Red, Blue, and Yellow Power Ranger each have 8 Power Gems\n",
      "    power_gems_red_blue_yellow = 8 * 3\n",
      "    #Pink and Green Power Ranger each have 2 Power Gems\n",
      "    power_gems_pink_green = 2 * 2\n",
      "    #The answer is\n",
      "    result = power_gems_red_blue_yellow - power_gems_pink_green\n",
      "    return result\n",
      "Question: Serena won 39 trophies. 25 of them are Grand Slam Trophies and the rest are Sports Illustrated Swimsuit Issue Trophies. How many Sports Illustrated Swimsuit Issue Trophies does she have?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Total trophies\n",
      "    total_trophies = 39\n",
      "    #Grand Slam Trophies\n",
      "    grand_slam_trophies = 25\n",
      "    #Sports Illustrated Swimsuit Issue Trophies\n",
      "    si_swiimsuit_trophies = total_trophies - grand_slam_trophies\n",
      "    return si_swiimsuit_trophies\n",
      "Question: Superman is 50 years old. He was 20 years old 10 years ago. How old is he in 5 years?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Superman is 50 years old\n",
      "    age_now = 50\n",
      "    #He was 20 years old 10 years ago\n",
      "    age_10_years_ago = 20\n",
      "    #10 years later, he was\n",
      "    age_20_years_ago = age_10_years_ago + 10\n",
      "    #In 5 years, he will be\n",
      "    age_in_5_years = age_20_years_ago + 5\n",
      "    return age_in_5_years\n",
      "Question: LeBron James scored 55 points in a game. He made 20 shots. 10 of them were from the field and 5 of them were from the 3-point line. How many free throws did he make?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #LeBron James scored 55 points in a game\n",
      "    points_total = 55\n",
      "    #He made 20 shots\n",
      "    shots_total = 20\n",
      "    #10 of them were from the field and 5 of them were from the 3-point line\n",
      "    shots_field = 10\n",
      "    shots_3_point = 5\n",
      "    #The answer is\n",
      "    shots_free_throw = shots_total - shots_field - shots_3_point\n",
      "    return shots_free_throw\n",
      "Question: There are 1000 cars in a parking lot. A fifth of the cars are Teslas. A third of the Teslas are white. How many of the Teslas are white?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #There are 1000 cars in a parking lot\n",
      "    cars = 1000\n",
      "    #A fifth of the cars are Teslas\n",
      "    teslas = cars / 5\n",
      "    #A third of the Teslas are white\n",
      "    white_teslas = teslas / 3\n",
      "    #The answer is\n",
      "    result = white_teslas\n",
      "    return result\n",
      "Question: There are 10 fish in the aquarium. Two-thirds of the fish are in the main aquarium and the rest are in the smaller aquarium. How many fish are in the smaller aquarium?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Total number of fish\n",
      "    total_fish = 10\n",
      "    #Two-thirds of the fish are in the main aquarium\n",
      "    fish_in_main_aquarium = 2/3 * total_fish\n",
      "    #The rest are in the smaller aquarium\n",
      "    fish_in_smaller_aquarium = total_fish - fish_in_main_aquarium\n",
      "    #The answer is\n",
      "    result = fish_in_smaller_aquarium\n",
      "    return result\n",
      "Question: Hello Kitty has 12 apples. She eats 3 of them, and the rest are divided into 3 equally-sized piles. How many apples are in each pile?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Hello Kitty has 12 apples\n",
      "    apples_initial = 12\n",
      "    #She eats 3 of them\n",
      "    apples_eaten = 3\n",
      "    #The rest are divided into 3 equally-sized piles\n",
      "    piles = 3\n",
      "    #The answer is\n",
      "    result = (apples_initial - apples_eaten) / piles\n",
      "    return result\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "for i in range(0, 10):\n",
    "    index = random.randint(0, len(df2))\n",
    "    question = df2.iloc[index]['question']\n",
    "    solution = df2.iloc[index]['solution']\n",
    "    print(f\"Question: {question}\\n\\nSolution:\\n{solution}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "d3d69bd6-491c-4373-b28a-844078d0dec9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: Batman is 55 years old. He was 25 years old when he became Batman. How long has he been Batman?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Batman is 55 years old\n",
      "    batman_age = 55\n",
      "    #He was 25 years old when he became Batman\n",
      "    batman_age_became = 25\n",
      "    #The answer is\n",
      "    result = batman_age - batman_age_became\n",
      "    return result\n",
      "Question: The Lacrosse team has 15 players.  They need 3 sticks each.  How many sticks do they need in total?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #The Lacrosse team has 15 players\n",
      "    players = 15\n",
      "    #They need 3 sticks each\n",
      "    sticks_per_player = 3\n",
      "    #The answer is\n",
      "    result = players * sticks_per_player\n",
      "    return result\n",
      "Question: Steph Curry is a basketball player. He made 50 three-pointers in the first game. He made 100 three-pointers in the second game. He made 200 three-pointers in the third game. How many three-pointers did he make in total?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Steph Curry made 50 three-pointers in the first game\n",
      "    three_pointers_first_game = 50\n",
      "    #He made 100 three-pointers in the second game\n",
      "    three_pointers_second_game = 100\n",
      "    #He made 200 three-pointers in the third game\n",
      "    three_pointers_third_game = 200\n",
      "    #The total is\n",
      "    result = three_pointers_first_game + three_pointers_second_game + three_pointers_third_game\n",
      "    return result\n",
      "Question: Batman is a superhero who fights crime in Gotham City. \n",
      "\n",
      "He has 100 Batarangs.  Each night, he uses 20 Batarangs. \n",
      "\n",
      "How many nights can he do this for?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Batman has 100 Batarangs\n",
      "    batarangs_initial = 100\n",
      "    #Each night, he uses 20 Batarangs\n",
      "    batarangs_per_night = 20\n",
      "    #The answer is\n",
      "    result = batarangs_initial // batarangs_per_night\n",
      "    return result\n",
      "Question: Ash Ketchum has 100 Pokémon. 20 of them are Pikachu. 40 of them are Squirtle. 30 of them are Bulbasaur. The rest are Pidgeotto. How many Pidgeotto does he have?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Ash Ketchum has 100 Pokémon\n",
      "    pokemon_total = 100\n",
      "    #20 of them are Pikachu\n",
      "    pikachu = 20\n",
      "    #40 of them are Squirtle\n",
      "    squirtle = 40\n",
      "    #30 of them are Bulbasaur\n",
      "    bulbasaur = 30\n",
      "    #The rest are Pidgeotto. The answer is\n",
      "    pidgeotto = pokemon_total - pikachu - squirtle - bulbasaur\n",
      "    return pidgeotto\n",
      "Question: A plane flies 1,000 miles per hour. \n",
      "It flies for 3 hours, \n",
      "Then it stops for 2 hours, \n",
      "And then it flies for 5 hours. \n",
      "How far has the plane flown?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #A plane flies 1,000 miles per hour\n",
      "    plane_speed = 1000\n",
      "    #It flies for 3 hours\n",
      "    flight_time_1 = 3\n",
      "    #Then it stops for 2 hours\n",
      "    stop_time = 2\n",
      "    #And then it flies for 5 hours\n",
      "    flight_time_2 = 5\n",
      "    #The total distance the plane has flown is\n",
      "    result = plane_speed * (flight_time_1 + flight_time_2)\n",
      "    return result\n",
      "Question: The Penguin has 2000 dollars. He spends 1000 dollars on a new umbrella and 500 dollars on a new suit. How much money does he have left?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #The Penguin started with 2000 dollars\n",
      "    money_initial = 2000\n",
      "    #He spent 1000 dollars on a new umbrella\n",
      "    umbrella_cost = 1000\n",
      "    #And 500 dollars on a new suit\n",
      "    suit_cost = 500\n",
      "    #The answer is\n",
      "    result = money_initial - umbrella_cost - suit_cost\n",
      "    return result\n",
      "Question: 124 pizza slices were left in the fridge. 32 of the slices had pepperoni, 42 of the slices had mushrooms, and 50 of the slices had both pepperoni and mushrooms. How many slices of pizza in the fridge did not have either of these toppings?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #124 pizza slices were left in the fridge\n",
      "    slices_total = 124\n",
      "    #32 of the slices had pepperoni\n",
      "    slices_pepperoni = 32\n",
      "    #42 of the slices had mushrooms\n",
      "    slices_mushrooms = 42\n",
      "    #50 of the slices had both pepperoni and mushrooms\n",
      "    slices_both = 50\n",
      "    #The answer is\n",
      "    result = slices_total - slices_pepperoni - slices_mushrooms + slices_both\n",
      "    return result\n",
      "Question: A scoop of vanilla ice cream is $3 and a scoop of chocolate ice cream is $4.  How much more does 3 scoops of chocolate ice cream cost than 3 scoops of vanilla ice cream?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #A scoop of vanilla ice cream is $3\n",
      "    vanilla_scoop = 3\n",
      "    #A scoop of chocolate ice cream is $4\n",
      "    chocolate_scoop = 4\n",
      "    #3 scoops of chocolate ice cream cost\n",
      "    chocolate_cost = 3 * chocolate_scoop\n",
      "    #3 scoops of vanilla ice cream cost\n",
      "    vanilla_cost = 3 * vanilla_scoop\n",
      "    #The answer is\n",
      "    result = chocolate_cost - vanilla_cost\n",
      "    return result\n",
      "Question: Captain Marvel can fly 1000 miles in 10 hours. She flies 500 miles to the first destination, 2000 to the second, and 1500 to the third. How many hours does it take her to fly to all three destinations?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Captain Marvel can fly 1000 miles in 10 hours\n",
      "    miles_per_hour = 1000 / 10\n",
      "    #She flies 500 miles to the first destination\n",
      "    miles_first_destination = 500\n",
      "    #2000 to the second\n",
      "    miles_second_destination = 2000\n",
      "    #1500 to the third\n",
      "    miles_third_destination = 1500\n",
      "    #The answer is\n",
      "    result = miles_first_destination / miles_per_hour + miles_second_destination / miles_per_hour + miles_third_destination / miles_per_hour\n",
      "    return result\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "for i in range(0, 10):\n",
    "    index = random.randint(0, len(df3))\n",
    "    question = df3.iloc[index]['question']\n",
    "    solution = df3.iloc[index]['solution']\n",
    "    print(f\"Question: {question}\\n\\nSolution:\\n{solution}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "93dfd344-b5ac-4fa0-8127-e3d1424aa433",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.85"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "17/20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69a81f38-40d4-4eeb-a611-daae88df0ddc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}