Parse_MATHWELL_Unannotated.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "17ae65ad-f11b-442a-b9b2-6b41013b7eb1",
   "metadata": {},
   "source": [
    "# Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8d610c0d-00fd-4ea8-b91d-ece6e872c721",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.stats as st"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74a1913a-3506-405c-852d-d80d0464b85f",
   "metadata": {},
   "source": [
    "# Parse First Text File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bb591ed4-de02-4f99-a272-cae18d1c29a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "Percentage of Questions with Executable Code: 55.395472995209715\n",
      "15149 27347\n"
     ]
    }
   ],
   "source": [
    "file_path = 'mathwell_questions2.txt'\n",
    "data = []\n",
    "\n",
    "with open(file_path, 'r') as file:\n",
    "    text = file.read()\n",
    "    sections = text.split(\"Topic: \")\n",
    "    #sections = text.split(\"### Response:\")\n",
    "    del sections[0]  # Remove the initial part before the question\n",
    "    total_questions = len(sections)\n",
    "    for section in sections:\n",
    "        try:\n",
    "            topic = section.split(\"### Response:\")[0].strip()\n",
    "            section = section.split(\"### Response:\")[1].strip()\n",
    "            question = section.split(\"Question:\")[1].strip()\n",
    "            question = question.split(\"Solution:\")[0].strip()\n",
    "            solution = section.split(\"Solution:\")[1].strip()\n",
    "            if \"##\" in question:\n",
    "                question = question.split(\"##\")[0]\n",
    "#             if \"\\n\\n\" in question:\n",
    "#                 question = question.split(\"\\n\\n\")\n",
    "#                 question = \"\\n\".join(question)\n",
    "            if \"\\nBel\" in solution:\n",
    "                solution = solution.split(\"\\nBel\")[0]\n",
    "            if \"##\" in solution:\n",
    "                solution = solution.split(\"##\")[0]\n",
    "            if \"return\" not in solution:\n",
    "                try: \n",
    "                    solution = solution + '\\n' + '    return result'\n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                            # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, 'topic': topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                                pass\n",
    "                except:\n",
    "                    pass\n",
    "            if \"return\" in solution:\n",
    "                try: \n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                        # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                            pass\n",
    "                except:\n",
    "                    pass\n",
    "        except:\n",
    "            pass\n",
    "df1 = pd.DataFrame.from_dict(data)\n",
    "result1 = \"Percentage of Questions with Executable Code: \" + str((len(df1)/total_questions)*100) #past percentage was 54.88\n",
    "print(result1)\n",
    "print(len(df1), total_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0bfde86c-6373-41c8-8ae0-84bbb1d47c46",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of questions that have 0 as answer: 6.9311505709947845\n"
     ]
    }
   ],
   "source": [
    "print(f\"Percentage of questions that have 0 as answer: {(len(df1[df1['answer']==0])/len(df1))*100}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "db0c1708-6816-4b66-ab72-0a441e43f023",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df1.drop_duplicates(subset='question')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8b271067-f922-4e3c-8b58-99a9e150b358",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A baker baked 120 cakes. He sold 40 on the fir...</td>\n",
       "      <td>def solution():\\n\\n    # The baker started wit...</td>\n",
       "      <td>30</td>\n",
       "      <td>Animals and Wildlife</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Naruto has 300 ramen noodles. He eats 20 ramen...</td>\n",
       "      <td>def solution():\\n    #Naruto has 300 ramen noo...</td>\n",
       "      <td>200</td>\n",
       "      <td>Naruto</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A 5th-grade class is doing a science project. ...</td>\n",
       "      <td>def solution():\\n    #20 students are in the c...</td>\n",
       "      <td>60</td>\n",
       "      <td>DIY Science Projects</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A mummy is 3000 years old. \\n\\nHis sarcophagus...</td>\n",
       "      <td>def solution():\\n    #The mummy is 3000 years ...</td>\n",
       "      <td>4000</td>\n",
       "      <td>Mummies</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The volleyball team has 12 players. \\n4 of the...</td>\n",
       "      <td>def solution():\\n    #Total number of players\\...</td>\n",
       "      <td>0</td>\n",
       "      <td>volleyball</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15143</th>\n",
       "      <td>A phoenix is a mythical bird that is reborn fr...</td>\n",
       "      <td>def solution():\\n    #The first phoenix was bo...</td>\n",
       "      <td>1005500</td>\n",
       "      <td>Mythology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15144</th>\n",
       "      <td>Ratatouille, the rat, ate 12 slices of pizza. ...</td>\n",
       "      <td>def solution():\\n    #Ratatouille ate 12 slice...</td>\n",
       "      <td>0</td>\n",
       "      <td>Ratatouille</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15145</th>\n",
       "      <td>A detective is investigating a murder. He inte...</td>\n",
       "      <td>def solution():\\n    #Total people interviewed...</td>\n",
       "      <td>20</td>\n",
       "      <td>Detectives and Mystery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15146</th>\n",
       "      <td>Shrek ate 30 onions. 10 of the onions were gre...</td>\n",
       "      <td>def solution():\\n    #Shrek ate 30 onions\\n   ...</td>\n",
       "      <td>0</td>\n",
       "      <td>Shrek</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15147</th>\n",
       "      <td>There are 24 My Little Ponies. \\n12 of them ar...</td>\n",
       "      <td>def solution():\\n    #There are 24 My Little P...</td>\n",
       "      <td>6</td>\n",
       "      <td>My Little Pony</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13972 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                question  \\\n",
       "0      A baker baked 120 cakes. He sold 40 on the fir...   \n",
       "1      Naruto has 300 ramen noodles. He eats 20 ramen...   \n",
       "2      A 5th-grade class is doing a science project. ...   \n",
       "3      A mummy is 3000 years old. \\n\\nHis sarcophagus...   \n",
       "4      The volleyball team has 12 players. \\n4 of the...   \n",
       "...                                                  ...   \n",
       "15143  A phoenix is a mythical bird that is reborn fr...   \n",
       "15144  Ratatouille, the rat, ate 12 slices of pizza. ...   \n",
       "15145  A detective is investigating a murder. He inte...   \n",
       "15146  Shrek ate 30 onions. 10 of the onions were gre...   \n",
       "15147  There are 24 My Little Ponies. \\n12 of them ar...   \n",
       "\n",
       "                                                solution   answer  \\\n",
       "0      def solution():\\n\\n    # The baker started wit...       30   \n",
       "1      def solution():\\n    #Naruto has 300 ramen noo...      200   \n",
       "2      def solution():\\n    #20 students are in the c...       60   \n",
       "3      def solution():\\n    #The mummy is 3000 years ...     4000   \n",
       "4      def solution():\\n    #Total number of players\\...        0   \n",
       "...                                                  ...      ...   \n",
       "15143  def solution():\\n    #The first phoenix was bo...  1005500   \n",
       "15144  def solution():\\n    #Ratatouille ate 12 slice...        0   \n",
       "15145  def solution():\\n    #Total people interviewed...       20   \n",
       "15146  def solution():\\n    #Shrek ate 30 onions\\n   ...        0   \n",
       "15147  def solution():\\n    #There are 24 My Little P...        6   \n",
       "\n",
       "                        topic  \n",
       "0        Animals and Wildlife  \n",
       "1                      Naruto  \n",
       "2        DIY Science Projects  \n",
       "3                     Mummies  \n",
       "4                  volleyball  \n",
       "...                       ...  \n",
       "15143               Mythology  \n",
       "15144             Ratatouille  \n",
       "15145  Detectives and Mystery  \n",
       "15146                   Shrek  \n",
       "15147          My Little Pony  \n",
       "\n",
       "[13972 rows x 4 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b3fd654-1ccf-4a9b-a998-09e9ce784989",
   "metadata": {},
   "source": [
    "# Parse Second Text File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a71cc7d1-2ac8-4556-ae4c-b67b3654524e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of Questions with Executable Code: 46.500577886220626\n",
      "7242 15574\n"
     ]
    }
   ],
   "source": [
    "file_path = 'mathwell_questions_no_topic.txt'\n",
    "data = []\n",
    "\n",
    "with open(file_path, 'r') as file:\n",
    "    text = file.read()\n",
    "    sections = text.split(\"Topic: \")\n",
    "    #sections = text.split(\"### Response:\")\n",
    "    del sections[0]  # Remove the initial part before the question\n",
    "    total_questions = len(sections)\n",
    "    for section in sections:\n",
    "        try:\n",
    "            topic = section.split(\"### Response:\")[0].strip()\n",
    "            section = section.split(\"### Response:\")[1].strip()\n",
    "            question = section.split(\"Question:\")[1].strip()\n",
    "            question = question.split(\"Solution:\")[0].strip()\n",
    "            solution = section.split(\"Solution:\")[1].strip()\n",
    "            if \"##\" in question:\n",
    "                question = question.split(\"##\")[0]\n",
    "#             if \"\\n\\n\" in question:\n",
    "#                 question = question.split(\"\\n\\n\")\n",
    "#                 question = \"\\n\".join(question)\n",
    "            if \"\\nBel\" in solution:\n",
    "                solution = solution.split(\"\\nBel\")[0]\n",
    "            if \"##\" in solution:\n",
    "                solution = solution.split(\"##\")[0]\n",
    "            if \"return\" not in solution:\n",
    "                try: \n",
    "                    solution = solution + '\\n' + '    return result'\n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                            # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, 'topic': topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                                pass\n",
    "                except:\n",
    "                    pass\n",
    "            if \"return\" in solution:\n",
    "                try: \n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                        # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                            pass\n",
    "                except:\n",
    "                    pass\n",
    "        except:\n",
    "            pass\n",
    "df2 = pd.DataFrame.from_dict(data)\n",
    "result1 = \"Percentage of Questions with Executable Code: \" + str((len(df2)/total_questions)*100)\n",
    "print(result1)\n",
    "print(len(df2), total_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bdd5bc93-4cce-40ab-a10d-b27cc200df8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of questions that have 0 as answer: 4.901960784313726\n"
     ]
    }
   ],
   "source": [
    "print(f\"Percentage of questions that have 0 as answer: {(len(df2[df2['answer']==0])/len(df2))*100}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "489f8c3b-03ff-471b-920b-25b3c7037c78",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df2.drop_duplicates(subset='question')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a0a6fa19-1a8a-44b6-ad5c-840bbd962b7f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2000 people in a stadium. 1/4 of the people ar...</td>\n",
       "      <td>def solution():\\n    #Total people in the stad...</td>\n",
       "      <td>850.0</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The first 500 people in line for the Comic-Con...</td>\n",
       "      <td>def solution():\\n    #The first 500 people in ...</td>\n",
       "      <td>1000</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>There are 120000 spiders in a spider farm. 100...</td>\n",
       "      <td>def solution():\\n    #There were 120000 spider...</td>\n",
       "      <td>100000</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A baker bakes 100 pies. He sells 20 of them on...</td>\n",
       "      <td>def solution():\\n    #Baker bakes 100 pies\\n  ...</td>\n",
       "      <td>10</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A 747 can hold 400 people. \\nA 777 can hold 30...</td>\n",
       "      <td>def solution():\\n    #A 747 can hold 400 peopl...</td>\n",
       "      <td>4000</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7232</th>\n",
       "      <td>400 people are at a rally. 200 of them are wea...</td>\n",
       "      <td>def solution():\\n    #Total people at the rall...</td>\n",
       "      <td>100</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7233</th>\n",
       "      <td>20000 people live in a small town. 2000 of the...</td>\n",
       "      <td>def solution():\\n    #12000 people live in a s...</td>\n",
       "      <td>0</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7237</th>\n",
       "      <td>A baker is baking 1000 cookies. He bakes 200 c...</td>\n",
       "      <td>def solution():\\n    #The baker is baking 1000...</td>\n",
       "      <td>-400</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7238</th>\n",
       "      <td>200 people are at a rally. 1/4 of them are wea...</td>\n",
       "      <td>def solution():\\n\\n    #Total number of people...</td>\n",
       "      <td>&lt;class 'int'&gt;</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7241</th>\n",
       "      <td>1000 people are at a rally. 200 of them are fr...</td>\n",
       "      <td>def solution():\\n    #1000 people are at a ral...</td>\n",
       "      <td>0</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4576 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question  \\\n",
       "0     2000 people in a stadium. 1/4 of the people ar...   \n",
       "1     The first 500 people in line for the Comic-Con...   \n",
       "2     There are 120000 spiders in a spider farm. 100...   \n",
       "3     A baker bakes 100 pies. He sells 20 of them on...   \n",
       "4     A 747 can hold 400 people. \\nA 777 can hold 30...   \n",
       "...                                                 ...   \n",
       "7232  400 people are at a rally. 200 of them are wea...   \n",
       "7233  20000 people live in a small town. 2000 of the...   \n",
       "7237  A baker is baking 1000 cookies. He bakes 200 c...   \n",
       "7238  200 people are at a rally. 1/4 of them are wea...   \n",
       "7241  1000 people are at a rally. 200 of them are fr...   \n",
       "\n",
       "                                               solution         answer  \\\n",
       "0     def solution():\\n    #Total people in the stad...          850.0   \n",
       "1     def solution():\\n    #The first 500 people in ...           1000   \n",
       "2     def solution():\\n    #There were 120000 spider...         100000   \n",
       "3     def solution():\\n    #Baker bakes 100 pies\\n  ...             10   \n",
       "4     def solution():\\n    #A 747 can hold 400 peopl...           4000   \n",
       "...                                                 ...            ...   \n",
       "7232  def solution():\\n    #Total people at the rall...            100   \n",
       "7233  def solution():\\n    #12000 people live in a s...              0   \n",
       "7237  def solution():\\n    #The baker is baking 1000...           -400   \n",
       "7238  def solution():\\n\\n    #Total number of people...  <class 'int'>   \n",
       "7241  def solution():\\n    #1000 people are at a ral...              0   \n",
       "\n",
       "            topic  \n",
       "0     unspecified  \n",
       "1     unspecified  \n",
       "2     unspecified  \n",
       "3     unspecified  \n",
       "4     unspecified  \n",
       "...           ...  \n",
       "7232  unspecified  \n",
       "7233  unspecified  \n",
       "7237  unspecified  \n",
       "7238  unspecified  \n",
       "7241  unspecified  \n",
       "\n",
       "[4576 rows x 4 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "951f5bb4-1202-4d77-b27d-3f04f79eca11",
   "metadata": {},
   "source": [
    "# Concat DFs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "34becf8f-3c44-42ec-b6d8-844ab69211f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "extra_qs = pd.read_csv('data/extra_qs.csv')\n",
    "df = pd.concat([df1, df2, extra_qs])\n",
    "df = df.drop_duplicates(subset='question')\n",
    "df = df.sample(frac = 1, random_state = 42)\n",
    "df = df[['question', 'solution', 'answer', 'topic']]\n",
    "#df = df[:17692]\n",
    "#df.to_csv(\"data/sgsm_unannotated.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "137893a8-9dab-4b07-9064-e5e463e96b18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10739</th>\n",
       "      <td>The Egyptians built 100 pyramids. 20 of them a...</td>\n",
       "      <td>def solution():\\n    #Total number of pyramids...</td>\n",
       "      <td>15</td>\n",
       "      <td>Ancient Civilizations</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5852</th>\n",
       "      <td>A 2-liter bottle of soda has 80 calories. A 1....</td>\n",
       "      <td>def solution():\\n    #A 2-liter bottle of soda...</td>\n",
       "      <td>400</td>\n",
       "      <td>Health and Nutrition</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1910</th>\n",
       "      <td>There are 20000 apples in an orchard. 2000 of ...</td>\n",
       "      <td>def solution():\\n    #There are 20000 apples i...</td>\n",
       "      <td>15000</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7809</th>\n",
       "      <td>There are 1000 pages in a Mystery Stories book...</td>\n",
       "      <td>def solution():\\n    #Total pages in the book\\...</td>\n",
       "      <td>500</td>\n",
       "      <td>Mystery Stories</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14961</th>\n",
       "      <td>1000 Minions are in a room. 200 of them are we...</td>\n",
       "      <td>def solution():\\n    #1000 Minions are in a ro...</td>\n",
       "      <td>1000</td>\n",
       "      <td>Minions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12196</th>\n",
       "      <td>The Ringling Bros. and Barnum &amp; Bailey Circus ...</td>\n",
       "      <td>def solution():\\n    #Total number of clowns\\n...</td>\n",
       "      <td>50</td>\n",
       "      <td>the Circus and Performing Arts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12932</th>\n",
       "      <td>A unicorn has 2000 magical hairs. 1000 of thes...</td>\n",
       "      <td>def solution():\\n    #A unicorn has 2000 magic...</td>\n",
       "      <td>500</td>\n",
       "      <td>Mythical Creatures</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5736</th>\n",
       "      <td>Fairy Gardens has 1000 tulips. 200 of the tuli...</td>\n",
       "      <td>def solution():\\n    #Total tulips\\n    total_...</td>\n",
       "      <td>0</td>\n",
       "      <td>Fairy Gardens</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>889</th>\n",
       "      <td>25% of the 1200 students at a high school want...</td>\n",
       "      <td>def solution():\\n    #1200 students at a high ...</td>\n",
       "      <td>180.0</td>\n",
       "      <td>Careers (e.g., firefighter, astronaut, doctor)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2690</th>\n",
       "      <td>200 people attended a rally. 150 of them were ...</td>\n",
       "      <td>def solution():\\n\\n    # total attendees\\n\\n  ...</td>\n",
       "      <td>0</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19306 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                question  \\\n",
       "10739  The Egyptians built 100 pyramids. 20 of them a...   \n",
       "5852   A 2-liter bottle of soda has 80 calories. A 1....   \n",
       "1910   There are 20000 apples in an orchard. 2000 of ...   \n",
       "7809   There are 1000 pages in a Mystery Stories book...   \n",
       "14961  1000 Minions are in a room. 200 of them are we...   \n",
       "...                                                  ...   \n",
       "12196  The Ringling Bros. and Barnum & Bailey Circus ...   \n",
       "12932  A unicorn has 2000 magical hairs. 1000 of thes...   \n",
       "5736   Fairy Gardens has 1000 tulips. 200 of the tuli...   \n",
       "889    25% of the 1200 students at a high school want...   \n",
       "2690   200 people attended a rally. 150 of them were ...   \n",
       "\n",
       "                                                solution answer  \\\n",
       "10739  def solution():\\n    #Total number of pyramids...     15   \n",
       "5852   def solution():\\n    #A 2-liter bottle of soda...    400   \n",
       "1910   def solution():\\n    #There are 20000 apples i...  15000   \n",
       "7809   def solution():\\n    #Total pages in the book\\...    500   \n",
       "14961  def solution():\\n    #1000 Minions are in a ro...   1000   \n",
       "...                                                  ...    ...   \n",
       "12196  def solution():\\n    #Total number of clowns\\n...     50   \n",
       "12932  def solution():\\n    #A unicorn has 2000 magic...    500   \n",
       "5736   def solution():\\n    #Total tulips\\n    total_...      0   \n",
       "889    def solution():\\n    #1200 students at a high ...  180.0   \n",
       "2690   def solution():\\n\\n    # total attendees\\n\\n  ...      0   \n",
       "\n",
       "                                                topic  \n",
       "10739                           Ancient Civilizations  \n",
       "5852                             Health and Nutrition  \n",
       "1910                                      unspecified  \n",
       "7809                                  Mystery Stories  \n",
       "14961                                         Minions  \n",
       "...                                               ...  \n",
       "12196                  the Circus and Performing Arts  \n",
       "12932                              Mythical Creatures  \n",
       "5736                                    Fairy Gardens  \n",
       "889    Careers (e.g., firefighter, astronaut, doctor)  \n",
       "2690                                      unspecified  \n",
       "\n",
       "[19306 rows x 4 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df ## last run had 17692"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "673e739b-5980-45e6-b592-cf91846045ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "      <th>subset</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...</td>\n",
       "      <td>def solution():\\n    #LeBron James is 6'9\" tal...</td>\n",
       "      <td>205.74</td>\n",
       "      <td>basketball</td>\n",
       "      <td>sgsm_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Captain America has 1200 ryo. He spends 400 ry...</td>\n",
       "      <td>def solution():\\n    #Captain America started ...</td>\n",
       "      <td>300.0</td>\n",
       "      <td>Captain America</td>\n",
       "      <td>sgsm_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3 scoops of ice cream cost $4. \\nHow much does...</td>\n",
       "      <td>def solution():\\n    #3 scoops of ice cream co...</td>\n",
       "      <td>13.333333333333332</td>\n",
       "      <td>ice cream</td>\n",
       "      <td>sgsm_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A volleyball team is 6 players. \\n3 of the pla...</td>\n",
       "      <td>def solution():\\n    #3 of the players are 16 ...</td>\n",
       "      <td>16.666666666666668</td>\n",
       "      <td>volleyball</td>\n",
       "      <td>sgsm_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Superman can fly at a speed of 1,000 miles per...</td>\n",
       "      <td>def solution():\\n    #Superman can fly at a sp...</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Superman</td>\n",
       "      <td>sgsm_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12196</th>\n",
       "      <td>The Ringling Bros. and Barnum &amp; Bailey Circus ...</td>\n",
       "      <td>def solution():\\n    #Total number of clowns\\n...</td>\n",
       "      <td>50</td>\n",
       "      <td>the Circus and Performing Arts</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12932</th>\n",
       "      <td>A unicorn has 2000 magical hairs. 1000 of thes...</td>\n",
       "      <td>def solution():\\n    #A unicorn has 2000 magic...</td>\n",
       "      <td>500</td>\n",
       "      <td>Mythical Creatures</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5736</th>\n",
       "      <td>Fairy Gardens has 1000 tulips. 200 of the tuli...</td>\n",
       "      <td>def solution():\\n    #Total tulips\\n    total_...</td>\n",
       "      <td>0</td>\n",
       "      <td>Fairy Gardens</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>889</th>\n",
       "      <td>25% of the 1200 students at a high school want...</td>\n",
       "      <td>def solution():\\n    #1200 students at a high ...</td>\n",
       "      <td>180.0</td>\n",
       "      <td>Careers (e.g., firefighter, astronaut, doctor)</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2690</th>\n",
       "      <td>200 people attended a rally. 150 of them were ...</td>\n",
       "      <td>def solution():\\n\\n    # total attendees\\n\\n  ...</td>\n",
       "      <td>0</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21399 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                question  \\\n",
       "0      LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...   \n",
       "1      Captain America has 1200 ryo. He spends 400 ry...   \n",
       "2      3 scoops of ice cream cost $4. \\nHow much does...   \n",
       "3      A volleyball team is 6 players. \\n3 of the pla...   \n",
       "4      Superman can fly at a speed of 1,000 miles per...   \n",
       "...                                                  ...   \n",
       "12196  The Ringling Bros. and Barnum & Bailey Circus ...   \n",
       "12932  A unicorn has 2000 magical hairs. 1000 of thes...   \n",
       "5736   Fairy Gardens has 1000 tulips. 200 of the tuli...   \n",
       "889    25% of the 1200 students at a high school want...   \n",
       "2690   200 people attended a rally. 150 of them were ...   \n",
       "\n",
       "                                                solution              answer  \\\n",
       "0      def solution():\\n    #LeBron James is 6'9\" tal...              205.74   \n",
       "1      def solution():\\n    #Captain America started ...               300.0   \n",
       "2      def solution():\\n    #3 scoops of ice cream co...  13.333333333333332   \n",
       "3      def solution():\\n    #3 of the players are 16 ...  16.666666666666668   \n",
       "4      def solution():\\n    #Superman can fly at a sp...                 0.2   \n",
       "...                                                  ...                 ...   \n",
       "12196  def solution():\\n    #Total number of clowns\\n...                  50   \n",
       "12932  def solution():\\n    #A unicorn has 2000 magic...                 500   \n",
       "5736   def solution():\\n    #Total tulips\\n    total_...                   0   \n",
       "889    def solution():\\n    #1200 students at a high ...               180.0   \n",
       "2690   def solution():\\n\\n    # total attendees\\n\\n  ...                   0   \n",
       "\n",
       "                                                topic            subset  \n",
       "0                                          basketball        sgsm_train  \n",
       "1                                     Captain America        sgsm_train  \n",
       "2                                           ice cream        sgsm_train  \n",
       "3                                          volleyball        sgsm_train  \n",
       "4                                            Superman        sgsm_train  \n",
       "...                                               ...               ...  \n",
       "12196                  the Circus and Performing Arts  sgsm_unannotated  \n",
       "12932                              Mythical Creatures  sgsm_unannotated  \n",
       "5736                                    Fairy Gardens  sgsm_unannotated  \n",
       "889    Careers (e.g., firefighter, astronaut, doctor)  sgsm_unannotated  \n",
       "2690                                      unspecified  sgsm_unannotated  \n",
       "\n",
       "[21399 rows x 5 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df = pd.read_csv('data/sgsm_unannotated.csv')\n",
    "def parse_solution(df):\n",
    "    if \"\\nBe\" in df['solution']:\n",
    "        solution = df['solution'].split(\"\\nBe\")[0]\n",
    "    if \"##\" in df['solution']:\n",
    "        solution = df['solution'].split(\"##\")[0]\n",
    "    else:\n",
    "        solution = df['solution']\n",
    "    return solution\n",
    "df['solution'] = df.apply(parse_solution, axis =1)\n",
    "df.to_csv(\"data/sgsm_unannotated.csv\")\n",
    "sgsm_train = pd.read_csv('data/sgsm_train.csv')\n",
    "# sgsm_train = sgsm_train.drop_duplicates(subset= 'question')\n",
    "# sgsm_train.to_csv('data/sgsm_train.csv')\n",
    "sgsm_train['subset'] = 'sgsm_train'\n",
    "df['subset'] = 'sgsm_unannotated'\n",
    "sgsm = pd.concat([sgsm_train, df])\n",
    "sgsm = sgsm[['question', 'solution', 'answer', 'topic', 'subset']]\n",
    "#sgsm.to_csv('data/sgsm.csv')\n",
    "#sgsm = sgsm.drop_duplicates(subset = ['question', 'subset'])\n",
    "sgsm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8db64314-f1fd-41ca-aa92-8e397a39a6ec",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "      <th>subset</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...</td>\n",
       "      <td>def solution():\\n    #LeBron James is 6'9\" tal...</td>\n",
       "      <td>205.74</td>\n",
       "      <td>basketball</td>\n",
       "      <td>sgsm_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Captain America has 1200 ryo. He spends 400 ry...</td>\n",
       "      <td>def solution():\\n    #Captain America started ...</td>\n",
       "      <td>300.0</td>\n",
       "      <td>Captain America</td>\n",
       "      <td>sgsm_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3 scoops of ice cream cost $4. \\nHow much does...</td>\n",
       "      <td>def solution():\\n    #3 scoops of ice cream co...</td>\n",
       "      <td>13.333333333333332</td>\n",
       "      <td>ice cream</td>\n",
       "      <td>sgsm_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A volleyball team is 6 players. \\n3 of the pla...</td>\n",
       "      <td>def solution():\\n    #3 of the players are 16 ...</td>\n",
       "      <td>16.666666666666668</td>\n",
       "      <td>volleyball</td>\n",
       "      <td>sgsm_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Superman can fly at a speed of 1,000 miles per...</td>\n",
       "      <td>def solution():\\n    #Superman can fly at a sp...</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Superman</td>\n",
       "      <td>sgsm_train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12196</th>\n",
       "      <td>The Ringling Bros. and Barnum &amp; Bailey Circus ...</td>\n",
       "      <td>def solution():\\n    #Total number of clowns\\n...</td>\n",
       "      <td>50</td>\n",
       "      <td>the Circus and Performing Arts</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12932</th>\n",
       "      <td>A unicorn has 2000 magical hairs. 1000 of thes...</td>\n",
       "      <td>def solution():\\n    #A unicorn has 2000 magic...</td>\n",
       "      <td>500</td>\n",
       "      <td>Mythical Creatures</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5736</th>\n",
       "      <td>Fairy Gardens has 1000 tulips. 200 of the tuli...</td>\n",
       "      <td>def solution():\\n    #Total tulips\\n    total_...</td>\n",
       "      <td>0</td>\n",
       "      <td>Fairy Gardens</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>889</th>\n",
       "      <td>25% of the 1200 students at a high school want...</td>\n",
       "      <td>def solution():\\n    #1200 students at a high ...</td>\n",
       "      <td>180.0</td>\n",
       "      <td>Careers (e.g., firefighter, astronaut, doctor)</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2690</th>\n",
       "      <td>200 people attended a rally. 150 of them were ...</td>\n",
       "      <td>def solution():\\n\\n    # total attendees\\n\\n  ...</td>\n",
       "      <td>0</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20490 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                question  \\\n",
       "0      LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...   \n",
       "1      Captain America has 1200 ryo. He spends 400 ry...   \n",
       "2      3 scoops of ice cream cost $4. \\nHow much does...   \n",
       "3      A volleyball team is 6 players. \\n3 of the pla...   \n",
       "4      Superman can fly at a speed of 1,000 miles per...   \n",
       "...                                                  ...   \n",
       "12196  The Ringling Bros. and Barnum & Bailey Circus ...   \n",
       "12932  A unicorn has 2000 magical hairs. 1000 of thes...   \n",
       "5736   Fairy Gardens has 1000 tulips. 200 of the tuli...   \n",
       "889    25% of the 1200 students at a high school want...   \n",
       "2690   200 people attended a rally. 150 of them were ...   \n",
       "\n",
       "                                                solution              answer  \\\n",
       "0      def solution():\\n    #LeBron James is 6'9\" tal...              205.74   \n",
       "1      def solution():\\n    #Captain America started ...               300.0   \n",
       "2      def solution():\\n    #3 scoops of ice cream co...  13.333333333333332   \n",
       "3      def solution():\\n    #3 of the players are 16 ...  16.666666666666668   \n",
       "4      def solution():\\n    #Superman can fly at a sp...                 0.2   \n",
       "...                                                  ...                 ...   \n",
       "12196  def solution():\\n    #Total number of clowns\\n...                  50   \n",
       "12932  def solution():\\n    #A unicorn has 2000 magic...                 500   \n",
       "5736   def solution():\\n    #Total tulips\\n    total_...                   0   \n",
       "889    def solution():\\n    #1200 students at a high ...               180.0   \n",
       "2690   def solution():\\n\\n    # total attendees\\n\\n  ...                   0   \n",
       "\n",
       "                                                topic            subset  \n",
       "0                                          basketball        sgsm_train  \n",
       "1                                     Captain America        sgsm_train  \n",
       "2                                           ice cream        sgsm_train  \n",
       "3                                          volleyball        sgsm_train  \n",
       "4                                            Superman        sgsm_train  \n",
       "...                                               ...               ...  \n",
       "12196                  the Circus and Performing Arts  sgsm_unannotated  \n",
       "12932                              Mythical Creatures  sgsm_unannotated  \n",
       "5736                                    Fairy Gardens  sgsm_unannotated  \n",
       "889    Careers (e.g., firefighter, astronaut, doctor)  sgsm_unannotated  \n",
       "2690                                      unspecified  sgsm_unannotated  \n",
       "\n",
       "[20490 rows x 5 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgsm = sgsm.drop_duplicates(subset = 'question')\n",
    "sgsm[sgsm['subset']=='sgsm_unannotated'].to_csv('data/sgsm_unannotated.csv')\n",
    "sgsm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "869ceece-35ab-4185-a587-47762d18bb70",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "subset\n",
       "sgsm_unannotated    18397\n",
       "sgsm_train           2093\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgsm.to_csv('data/sgsm.csv', index=False)\n",
    "sgsm.to_csv('sgsm.csv', index=False)\n",
    "sgsm['subset'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "52f21bb8-bb35-404a-a309-fad4cf9e3a49",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...</td>\n",
       "      <td>def solution():\\n    #LeBron James is 6'9\" tal...</td>\n",
       "      <td>205.74</td>\n",
       "      <td>basketball</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Captain America has 1200 ryo. He spends 400 ry...</td>\n",
       "      <td>def solution():\\n    #Captain America started ...</td>\n",
       "      <td>300.0</td>\n",
       "      <td>Captain America</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3 scoops of ice cream cost $4. \\nHow much does...</td>\n",
       "      <td>def solution():\\n    #3 scoops of ice cream co...</td>\n",
       "      <td>13.333333333333332</td>\n",
       "      <td>ice cream</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>A volleyball team is 6 players. \\n3 of the pla...</td>\n",
       "      <td>def solution():\\n    #3 of the players are 16 ...</td>\n",
       "      <td>16.666666666666668</td>\n",
       "      <td>volleyball</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>Superman can fly at a speed of 1,000 miles per...</td>\n",
       "      <td>def solution():\\n    #Superman can fly at a sp...</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Superman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2088</th>\n",
       "      <td>1901</td>\n",
       "      <td>30 unicorns are in a herd. 10 unicorns in the ...</td>\n",
       "      <td>def solution():\\n    #Total number of unicorns...</td>\n",
       "      <td>17</td>\n",
       "      <td>unicorns</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2089</th>\n",
       "      <td>1902</td>\n",
       "      <td>A pizza has 8 slices. 4 of the slices have pep...</td>\n",
       "      <td>def solution():\\n    #A pizza has 8 slices\\n  ...</td>\n",
       "      <td>4</td>\n",
       "      <td>pizza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2090</th>\n",
       "      <td>1903</td>\n",
       "      <td>12000 people were at a rally. 10000 of them we...</td>\n",
       "      <td>def solution():\\n    #Total number of people a...</td>\n",
       "      <td>500</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2091</th>\n",
       "      <td>1904</td>\n",
       "      <td>A bakery made 300 muffins. 150 of the muffins ...</td>\n",
       "      <td>def solution():\\n    #Total muffins\\n    total...</td>\n",
       "      <td>60</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2092</th>\n",
       "      <td>1905</td>\n",
       "      <td>A pizza is cut into 12 slices. 5 slices are to...</td>\n",
       "      <td>def solution():\\n    #A pizza is cut into 12 s...</td>\n",
       "      <td>6</td>\n",
       "      <td>pizza</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2093 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                                           question  \\\n",
       "0              0  LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...   \n",
       "1              1  Captain America has 1200 ryo. He spends 400 ry...   \n",
       "2              3  3 scoops of ice cream cost $4. \\nHow much does...   \n",
       "3              5  A volleyball team is 6 players. \\n3 of the pla...   \n",
       "4              6  Superman can fly at a speed of 1,000 miles per...   \n",
       "...          ...                                                ...   \n",
       "2088        1901  30 unicorns are in a herd. 10 unicorns in the ...   \n",
       "2089        1902  A pizza has 8 slices. 4 of the slices have pep...   \n",
       "2090        1903  12000 people were at a rally. 10000 of them we...   \n",
       "2091        1904  A bakery made 300 muffins. 150 of the muffins ...   \n",
       "2092        1905  A pizza is cut into 12 slices. 5 slices are to...   \n",
       "\n",
       "                                               solution              answer  \\\n",
       "0     def solution():\\n    #LeBron James is 6'9\" tal...              205.74   \n",
       "1     def solution():\\n    #Captain America started ...               300.0   \n",
       "2     def solution():\\n    #3 scoops of ice cream co...  13.333333333333332   \n",
       "3     def solution():\\n    #3 of the players are 16 ...  16.666666666666668   \n",
       "4     def solution():\\n    #Superman can fly at a sp...                 0.2   \n",
       "...                                                 ...                 ...   \n",
       "2088  def solution():\\n    #Total number of unicorns...                  17   \n",
       "2089  def solution():\\n    #A pizza has 8 slices\\n  ...                   4   \n",
       "2090  def solution():\\n    #Total number of people a...                 500   \n",
       "2091  def solution():\\n    #Total muffins\\n    total...                  60   \n",
       "2092  def solution():\\n    #A pizza is cut into 12 s...                   6   \n",
       "\n",
       "                topic  \n",
       "0          basketball  \n",
       "1     Captain America  \n",
       "2           ice cream  \n",
       "3          volleyball  \n",
       "4            Superman  \n",
       "...               ...  \n",
       "2088         unicorns  \n",
       "2089            pizza  \n",
       "2090      unspecified  \n",
       "2091      unspecified  \n",
       "2092            pizza  \n",
       "\n",
       "[2093 rows x 5 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgsm_train = pd.read_csv('data/sgsm_train.csv')\n",
    "sgsm_train.drop_duplicates(subset= ['question', 'solution'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c68d8c5c-c45a-41e6-835e-630462c5e72c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...</td>\n",
       "      <td>def solution():\\n    #LeBron James is 6'9\" tal...</td>\n",
       "      <td>205.74</td>\n",
       "      <td>basketball</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Captain America has 1200 ryo. He spends 400 ry...</td>\n",
       "      <td>def solution():\\n    #Captain America started ...</td>\n",
       "      <td>300.0</td>\n",
       "      <td>Captain America</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3 scoops of ice cream cost $4. \\nHow much does...</td>\n",
       "      <td>def solution():\\n    #3 scoops of ice cream co...</td>\n",
       "      <td>13.333333333333332</td>\n",
       "      <td>ice cream</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>A volleyball team is 6 players. \\n3 of the pla...</td>\n",
       "      <td>def solution():\\n    #3 of the players are 16 ...</td>\n",
       "      <td>16.666666666666668</td>\n",
       "      <td>volleyball</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>Superman can fly at a speed of 1,000 miles per...</td>\n",
       "      <td>def solution():\\n    #Superman can fly at a sp...</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Superman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2088</th>\n",
       "      <td>1901</td>\n",
       "      <td>30 unicorns are in a herd. 10 unicorns in the ...</td>\n",
       "      <td>def solution():\\n    #Total number of unicorns...</td>\n",
       "      <td>17</td>\n",
       "      <td>unicorns</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2089</th>\n",
       "      <td>1902</td>\n",
       "      <td>A pizza has 8 slices. 4 of the slices have pep...</td>\n",
       "      <td>def solution():\\n    #A pizza has 8 slices\\n  ...</td>\n",
       "      <td>4</td>\n",
       "      <td>pizza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2090</th>\n",
       "      <td>1903</td>\n",
       "      <td>12000 people were at a rally. 10000 of them we...</td>\n",
       "      <td>def solution():\\n    #Total number of people a...</td>\n",
       "      <td>500</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2091</th>\n",
       "      <td>1904</td>\n",
       "      <td>A bakery made 300 muffins. 150 of the muffins ...</td>\n",
       "      <td>def solution():\\n    #Total muffins\\n    total...</td>\n",
       "      <td>60</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2092</th>\n",
       "      <td>1905</td>\n",
       "      <td>A pizza is cut into 12 slices. 5 slices are to...</td>\n",
       "      <td>def solution():\\n    #A pizza is cut into 12 s...</td>\n",
       "      <td>6</td>\n",
       "      <td>pizza</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2093 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                                           question  \\\n",
       "0              0  LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...   \n",
       "1              1  Captain America has 1200 ryo. He spends 400 ry...   \n",
       "2              3  3 scoops of ice cream cost $4. \\nHow much does...   \n",
       "3              5  A volleyball team is 6 players. \\n3 of the pla...   \n",
       "4              6  Superman can fly at a speed of 1,000 miles per...   \n",
       "...          ...                                                ...   \n",
       "2088        1901  30 unicorns are in a herd. 10 unicorns in the ...   \n",
       "2089        1902  A pizza has 8 slices. 4 of the slices have pep...   \n",
       "2090        1903  12000 people were at a rally. 10000 of them we...   \n",
       "2091        1904  A bakery made 300 muffins. 150 of the muffins ...   \n",
       "2092        1905  A pizza is cut into 12 slices. 5 slices are to...   \n",
       "\n",
       "                                               solution              answer  \\\n",
       "0     def solution():\\n    #LeBron James is 6'9\" tal...              205.74   \n",
       "1     def solution():\\n    #Captain America started ...               300.0   \n",
       "2     def solution():\\n    #3 scoops of ice cream co...  13.333333333333332   \n",
       "3     def solution():\\n    #3 of the players are 16 ...  16.666666666666668   \n",
       "4     def solution():\\n    #Superman can fly at a sp...                 0.2   \n",
       "...                                                 ...                 ...   \n",
       "2088  def solution():\\n    #Total number of unicorns...                  17   \n",
       "2089  def solution():\\n    #A pizza has 8 slices\\n  ...                   4   \n",
       "2090  def solution():\\n    #Total number of people a...                 500   \n",
       "2091  def solution():\\n    #Total muffins\\n    total...                  60   \n",
       "2092  def solution():\\n    #A pizza is cut into 12 s...                   6   \n",
       "\n",
       "                topic  \n",
       "0          basketball  \n",
       "1     Captain America  \n",
       "2           ice cream  \n",
       "3          volleyball  \n",
       "4            Superman  \n",
       "...               ...  \n",
       "2088         unicorns  \n",
       "2089            pizza  \n",
       "2090      unspecified  \n",
       "2091      unspecified  \n",
       "2092            pizza  \n",
       "\n",
       "[2093 rows x 5 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgsm_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "038a2267-fcd1-46c7-a184-8715698abd36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "      <th>subset</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10739</th>\n",
       "      <td>The Egyptians built 100 pyramids. 20 of them a...</td>\n",
       "      <td>def solution():\\n    #Total number of pyramids...</td>\n",
       "      <td>15</td>\n",
       "      <td>Ancient Civilizations</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5852</th>\n",
       "      <td>A 2-liter bottle of soda has 80 calories. A 1....</td>\n",
       "      <td>def solution():\\n    #A 2-liter bottle of soda...</td>\n",
       "      <td>400</td>\n",
       "      <td>Health and Nutrition</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1910</th>\n",
       "      <td>There are 20000 apples in an orchard. 2000 of ...</td>\n",
       "      <td>def solution():\\n    #There are 20000 apples i...</td>\n",
       "      <td>15000</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7809</th>\n",
       "      <td>There are 1000 pages in a Mystery Stories book...</td>\n",
       "      <td>def solution():\\n    #Total pages in the book\\...</td>\n",
       "      <td>500</td>\n",
       "      <td>Mystery Stories</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14961</th>\n",
       "      <td>1000 Minions are in a room. 200 of them are we...</td>\n",
       "      <td>def solution():\\n    #1000 Minions are in a ro...</td>\n",
       "      <td>1000</td>\n",
       "      <td>Minions</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12196</th>\n",
       "      <td>The Ringling Bros. and Barnum &amp; Bailey Circus ...</td>\n",
       "      <td>def solution():\\n    #Total number of clowns\\n...</td>\n",
       "      <td>50</td>\n",
       "      <td>the Circus and Performing Arts</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12932</th>\n",
       "      <td>A unicorn has 2000 magical hairs. 1000 of thes...</td>\n",
       "      <td>def solution():\\n    #A unicorn has 2000 magic...</td>\n",
       "      <td>500</td>\n",
       "      <td>Mythical Creatures</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5736</th>\n",
       "      <td>Fairy Gardens has 1000 tulips. 200 of the tuli...</td>\n",
       "      <td>def solution():\\n    #Total tulips\\n    total_...</td>\n",
       "      <td>0</td>\n",
       "      <td>Fairy Gardens</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>889</th>\n",
       "      <td>25% of the 1200 students at a high school want...</td>\n",
       "      <td>def solution():\\n    #1200 students at a high ...</td>\n",
       "      <td>180.0</td>\n",
       "      <td>Careers (e.g., firefighter, astronaut, doctor)</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2690</th>\n",
       "      <td>200 people attended a rally. 150 of them were ...</td>\n",
       "      <td>def solution():\\n\\n    # total attendees\\n\\n  ...</td>\n",
       "      <td>0</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>sgsm_unannotated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19306 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                question  \\\n",
       "10739  The Egyptians built 100 pyramids. 20 of them a...   \n",
       "5852   A 2-liter bottle of soda has 80 calories. A 1....   \n",
       "1910   There are 20000 apples in an orchard. 2000 of ...   \n",
       "7809   There are 1000 pages in a Mystery Stories book...   \n",
       "14961  1000 Minions are in a room. 200 of them are we...   \n",
       "...                                                  ...   \n",
       "12196  The Ringling Bros. and Barnum & Bailey Circus ...   \n",
       "12932  A unicorn has 2000 magical hairs. 1000 of thes...   \n",
       "5736   Fairy Gardens has 1000 tulips. 200 of the tuli...   \n",
       "889    25% of the 1200 students at a high school want...   \n",
       "2690   200 people attended a rally. 150 of them were ...   \n",
       "\n",
       "                                                solution answer  \\\n",
       "10739  def solution():\\n    #Total number of pyramids...     15   \n",
       "5852   def solution():\\n    #A 2-liter bottle of soda...    400   \n",
       "1910   def solution():\\n    #There are 20000 apples i...  15000   \n",
       "7809   def solution():\\n    #Total pages in the book\\...    500   \n",
       "14961  def solution():\\n    #1000 Minions are in a ro...   1000   \n",
       "...                                                  ...    ...   \n",
       "12196  def solution():\\n    #Total number of clowns\\n...     50   \n",
       "12932  def solution():\\n    #A unicorn has 2000 magic...    500   \n",
       "5736   def solution():\\n    #Total tulips\\n    total_...      0   \n",
       "889    def solution():\\n    #1200 students at a high ...  180.0   \n",
       "2690   def solution():\\n\\n    # total attendees\\n\\n  ...      0   \n",
       "\n",
       "                                                topic            subset  \n",
       "10739                           Ancient Civilizations  sgsm_unannotated  \n",
       "5852                             Health and Nutrition  sgsm_unannotated  \n",
       "1910                                      unspecified  sgsm_unannotated  \n",
       "7809                                  Mystery Stories  sgsm_unannotated  \n",
       "14961                                         Minions  sgsm_unannotated  \n",
       "...                                               ...               ...  \n",
       "12196                  the Circus and Performing Arts  sgsm_unannotated  \n",
       "12932                              Mythical Creatures  sgsm_unannotated  \n",
       "5736                                    Fairy Gardens  sgsm_unannotated  \n",
       "889    Careers (e.g., firefighter, astronaut, doctor)  sgsm_unannotated  \n",
       "2690                                      unspecified  sgsm_unannotated  \n",
       "\n",
       "[19306 rows x 5 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "125969fd-e204-451b-9a08-5ca30bf718f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "topic\n",
       "unspecified                                          4383\n",
       "National Parks                                        159\n",
       "LeBron James                                          157\n",
       "Dance and Ballet                                      154\n",
       "cheer leading                                         154\n",
       "                                                     ... \n",
       "Transformers                                           18\n",
       "Up                                                     18\n",
       "Pokemon                                                13\n",
       "Hobbies (e.g., stamp collecting, coin collecting)      10\n",
       "Pets (cats, dogs, hamsters)                             6\n",
       "Name: count, Length: 198, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['topic'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "483b9885-36e2-404d-8336-f858934e11a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: A soccer team has 11 players. 3 of them are goalkeepers, 4 are defenders, and 4 are forwards. How many midfielders does the team have?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Total number of players\n",
      "    total_players = 11\n",
      "    #Number of goalkeepers, defenders, and forwards\n",
      "    goalkeepers = 3\n",
      "    defenders = 4\n",
      "    forwards = 4\n",
      "    #The answer is\n",
      "    midfielders = total_players - goalkeepers - defenders - forwards\n",
      "    return midfielders\n",
      "Question: There are 1200 volcanoes on Earth. \n",
      "\n",
      "600 of them are active, \n",
      "\n",
      "300 of them are dormant, \n",
      "\n",
      "and 300 of them are extinct. \n",
      "\n",
      "How many volcanoes are there in each category?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "\n",
      "    #Total number of volcanoes\n",
      "\n",
      "    total_volcanoes = 1200\n",
      "\n",
      "    #Number of active volcanoes\n",
      "\n",
      "    active_volcanoes = 600\n",
      "\n",
      "    #Number of dormant volcanoes\n",
      "\n",
      "    dormant_volcanoes = 300\n",
      "\n",
      "    #Number of extinct volcanoes\n",
      "\n",
      "    extinct_volcanoes = 300\n",
      "\n",
      "    #The answer is\n",
      "\n",
      "    result = [active_volcanoes, dormant_volcanoes, extinct_volcanoes]\n",
      "\n",
      "    return result\n",
      "Question: An artist has 100 canvases. \n",
      "He uses 20 canvases to paint a series of landscapes. \n",
      "He uses 30 canvases to paint a series of portraits. \n",
      "He uses 10 canvases to paint a series of still lifes. \n",
      "How many canvases does he have left?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #The artist has 100 canvases\n",
      "    canvases = 100\n",
      "    #He uses 20 canvases to paint a series of landscapes\n",
      "    landscapes = 20\n",
      "    #30 canvases to paint a series of portraits\n",
      "    portraits = 30\n",
      "    #10 canvases to paint a series of still lifes\n",
      "    still_lifes = 10\n",
      "    #The answer is\n",
      "    result = canvases - landscapes - portraits - still_lifes\n",
      "    return result\n",
      "Question: 10000 people are at a rally. 2000 more people show up at the rally. 3000 people at the rally are under the age of 18. 4000 people at the rally are over the age of 65. How many people at the rally are between the ages of 18 and 65?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Total number of people at the rally\n",
      "    total_people = 10000\n",
      "    #2000 more people show up\n",
      "    more_people = 2000\n",
      "    #3000 people are under the age of 18\n",
      "    under_18 = 3000\n",
      "    #4000 people are over the age of 65\n",
      "    over_65 = 4000\n",
      "    #The answer is\n",
      "    result = total_people + more_people - under_18 - over_65\n",
      "    return result\n",
      "Question: 200 people are at a rally. 120 of them are wearing red hats. 60 of them are wearing blue hats. 20 of them are wearing both red and blue hats. How many people are not wearing a red or blue hat?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Total people at the rally\n",
      "    total_people = 200\n",
      "    #People wearing red hats\n",
      "    red_hats = 120\n",
      "    #People wearing blue hats\n",
      "    blue_hats = 60\n",
      "    #People wearing both red and blue hats\n",
      "    both_hats = 20\n",
      "    #People not wearing a red or blue hat\n",
      "    other_hats = total_people - red_hats - blue_hats + both_hats\n",
      "    return other_hats\n",
      "Question: A farmer has 100 cows. 50 of the cows are Holsteins, 30 are Jerseys, and the rest are Guernseys. How many Guernseys does the farmer have?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Total number of cows\n",
      "    total_cows = 100\n",
      "    #Number of Holsteins\n",
      "    holsteins = 50\n",
      "    #Number of Jerseys\n",
      "    jerseys = 30\n",
      "    #The rest are Guernseys. The answer is\n",
      "    guernseys = total_cows - holsteins - jerseys\n",
      "    return guernseys\n",
      "Question: A farmer has 500 chickens. 200 of them are hens. 100 of the hens have 5 chicks each. 20 of the hens have 3 chicks each. How many chickens does the farmer have in total?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Total chickens\n",
      "    total_chickens = 500\n",
      "    #Hens\n",
      "    hens = 200\n",
      "    #Hens with 5 chicks each\n",
      "    hens_5_chicks = 100\n",
      "    #Hens with 3 chicks each\n",
      "    hens_3_chicks = 20\n",
      "    #The answer is\n",
      "    result = total_chickens + hens_5_chicks * 5 + hens_3_chicks * 3\n",
      "    return result\n",
      "Question: Ash has 400 Pokémon. \n",
      "\n",
      "200 of them are in the Pokémon Center, 100 of them are in the PC, and the rest are in his party. \n",
      "\n",
      "How many Pokémon are in his party?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "\n",
      "  # total number of Pokémon Ash has\n",
      "  pokemon_total = 400\n",
      "\n",
      "  # number of Pokémon in the Pokémon Center\n",
      "  pokemon_center = 200\n",
      "\n",
      "  # number of Pokémon in the PC\n",
      "  pokemon_pc = 100\n",
      "\n",
      "  # number of Pokémon in his party\n",
      "  pokemon_party = pokemon_total - pokemon_center - pokemon_pc\n",
      "\n",
      "  # return the answer\n",
      "  return pokemon_party\n",
      "Question: A candy store has 400 lollipops. 120 of them are cherry, 170 of them are strawberry, and 110 of them are lemon. How many of them are not one of these three?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #The candy store has 400 lollipops\n",
      "    lollipops_total = 400\n",
      "    #120 of them are cherry\n",
      "    lollipops_cherry = 120\n",
      "    #170 of them are strawberry\n",
      "    lollipops_strawberry = 170\n",
      "    #110 of them are lemon\n",
      "    lollipops_lemon = 110\n",
      "    #The answer is\n",
      "    result = lollipops_total - lollipops_cherry - lollipops_strawberry - lollipops_lemon\n",
      "    return result\n",
      "Question: A factory has 100 robots. 20% of the robots are for welding, 30% of the robots are for assembly, and the rest are for inspection. How many robots are for inspection?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #The factory has 100 robots\n",
      "    robots_total = 100\n",
      "    #20% of the robots are for welding\n",
      "    welding_robots = robots_total * 0.20\n",
      "    #30% of the robots are for assembly\n",
      "    assembly_robots = robots_total * 0.30\n",
      "    #The rest are for inspection\n",
      "    inspection_robots = robots_total - welding_robots - assembly_robots\n",
      "    #The answer is\n",
      "    result = inspection_robots\n",
      "    return result\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "for i in range(0, 10):\n",
    "    index = random.randint(0, len(df))\n",
    "    question = df.iloc[index]['question']\n",
    "    solution = df.iloc[index]['solution']\n",
    "    print(f\"Question: {question}\\n\\nSolution:\\n{solution}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a0e102d7-6c87-4b43-8143-5bd7f9ef7968",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20490"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "18397+2093"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9beecae7-7672-4a82-aac1-6302dd9d4def",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import random\n",
    "# Read the content of the first JSON file\n",
    "with open('data/mathwell_final.json', 'r') as file:\n",
    "    data1 = json.load(file)\n",
    "\n",
    "# Read the content of the second JSON file\n",
    "with open('data/initial_train.json', 'r') as file:\n",
    "    data2 = json.load(file)\n",
    "\n",
    "# Merge the contents into a single list (assuming both files contain lists)\n",
    "merged_data = data1 + data2\n",
    "\n",
    "# Shuffle the merged data\n",
    "random.shuffle(merged_data)\n",
    "\n",
    "# Write the shuffled data to a new JSON file\n",
    "with open('data/full_train.json', 'w') as file:\n",
    "    json.dump(merged_data, file)  # Use indent for pretty formatting (optional)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ed0466b-ce09-4422-b244-773c7d6efecc",
   "metadata": {},
   "source": [
    "## Format Annotation Data for KTO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e3daf738-187c-44ed-a809-97ff2e38d21b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>solvability</th>\n",
       "      <th>appropriateness</th>\n",
       "      <th>mac</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>llama</td>\n",
       "      <td>Steph Curry has 3000 points. 1000 of them are ...</td>\n",
       "      <td>def solution():\\n    #Steph Curry has 3000 poi...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mammoth</td>\n",
       "      <td>The soccer team has 15 players. Each player ha...</td>\n",
       "      <td>def solution():\\n    #Number of players\\n    p...</td>\n",
       "      <td>30</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>llama</td>\n",
       "      <td>A commercial airplane takes 12 hours to fly fr...</td>\n",
       "      <td>def solution():\\n    #A commercial airplane ta...</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>llema</td>\n",
       "      <td>In the original comic book series, the four Tu...</td>\n",
       "      <td>def solution():\\n    #Raphael is 6 inches tall...</td>\n",
       "      <td>30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>llema</td>\n",
       "      <td>Spiderman has a web-shooter that shoots spider...</td>\n",
       "      <td>def solution():\\n    #Spiderman needs to load ...</td>\n",
       "      <td>180</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4729</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>A field hockey game has 11 players per team. T...</td>\n",
       "      <td>def solution():\\n    #A field hockey game has ...</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4730</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>100 turtles are at the pond. 10 turtles are ea...</td>\n",
       "      <td>def solution():\\n    #Total turtles\\n    total...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4731</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>Barbie has 1200 Instagram followers. 200 more ...</td>\n",
       "      <td>def solution():\\n    #Barbie started with 1200...</td>\n",
       "      <td>1150</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4732</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>1234567890 is a 10-digit number. \\nHow many 3-...</td>\n",
       "      <td>def solution():\\n    #10-digit number has 10 d...</td>\n",
       "      <td>3.3333333333333335</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4733</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>7 cats and 23 dogs are in a cage. 14 of the an...</td>\n",
       "      <td>def solution():\\n    #Total cats and dogs\\n   ...</td>\n",
       "      <td>13</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4734 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                model                                           question  \\\n",
       "0               llama  Steph Curry has 3000 points. 1000 of them are ...   \n",
       "1             mammoth  The soccer team has 15 players. Each player ha...   \n",
       "2               llama  A commercial airplane takes 12 hours to fly fr...   \n",
       "3               llema  In the original comic book series, the four Tu...   \n",
       "4               llema  Spiderman has a web-shooter that shoots spider...   \n",
       "...               ...                                                ...   \n",
       "4729  finetuned_llama  A field hockey game has 11 players per team. T...   \n",
       "4730  finetuned_llama  100 turtles are at the pond. 10 turtles are ea...   \n",
       "4731  finetuned_llama  Barbie has 1200 Instagram followers. 200 more ...   \n",
       "4732  finetuned_llama  1234567890 is a 10-digit number. \\nHow many 3-...   \n",
       "4733  finetuned_llama  7 cats and 23 dogs are in a cage. 14 of the an...   \n",
       "\n",
       "                                               solution              answer  \\\n",
       "0     def solution():\\n    #Steph Curry has 3000 poi...                   0   \n",
       "1     def solution():\\n    #Number of players\\n    p...                  30   \n",
       "2     def solution():\\n    #A commercial airplane ta...                 0.6   \n",
       "3     def solution():\\n    #Raphael is 6 inches tall...                  30   \n",
       "4     def solution():\\n    #Spiderman needs to load ...                 180   \n",
       "...                                                 ...                 ...   \n",
       "4729  def solution():\\n    #A field hockey game has ...                  33   \n",
       "4730  def solution():\\n    #Total turtles\\n    total...                   0   \n",
       "4731  def solution():\\n    #Barbie started with 1200...                1150   \n",
       "4732  def solution():\\n    #10-digit number has 10 d...  3.3333333333333335   \n",
       "4733  def solution():\\n    #Total cats and dogs\\n   ...                  13   \n",
       "\n",
       "      accuracy  solvability  appropriateness  mac  \n",
       "0          1.0            1              1.0    1  \n",
       "1          1.0            1              1.0    1  \n",
       "2          0.0            1              1.0    0  \n",
       "3          NaN            0              NaN    0  \n",
       "4          1.0            1              1.0    1  \n",
       "...        ...          ...              ...  ...  \n",
       "4729       0.0            1              0.0    0  \n",
       "4730       0.0            1              0.0    0  \n",
       "4731       0.0            0              0.0    0  \n",
       "4732       0.0            1              0.0    0  \n",
       "4733       0.0            0              0.0    0  \n",
       "\n",
       "[4734 rows x 8 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotations = pd.read_csv('data/annotations.csv')\n",
    "annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "343d5e78-c89e-4223-8986-500df6ad941e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
      "\n",
      "### Instruction:\n",
      "Write a grade school math word problem and Python function with a commented out step-by-step solution to solve the word problem.\n",
      "\n",
      "### Response: \n"
     ]
    }
   ],
   "source": [
    "prompt = \"Write a grade school math word problem and Python function with a commented out step-by-step solution to solve the word problem.\"\n",
    "formatted_prompt = [f\"Below is an instruction that describes a task. \" \n",
    "                f\"Write a response that appropriately completes the request.\\n\\n\" + \n",
    "                f\"### Instruction:\\n{prompt}\\n\\n### Response: \"]\n",
    "formatted_prompt = \"\".join(formatted_prompt)\n",
    "print(formatted_prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "a6e2bcd6-a4f5-4226-adf2-413445e25086",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>solvability</th>\n",
       "      <th>appropriateness</th>\n",
       "      <th>mac</th>\n",
       "      <th>prompt</th>\n",
       "      <th>completion</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2307</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>The Varsity Lacrosse team has 24 players. The ...</td>\n",
       "      <td>def solution():\\n    #The Varsity Lacrosse tea...</td>\n",
       "      <td>30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: The Varsity Lacrosse team has 24 pla...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>354</th>\n",
       "      <td>llema</td>\n",
       "      <td>If the Super Mario can jump over a total of 5 ...</td>\n",
       "      <td>def solution():\\n    #The super Mario can jump...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: If the Super Mario can jump over a t...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1397</th>\n",
       "      <td>gpt4</td>\n",
       "      <td>Naruto has been asked to deliver scrolls to hi...</td>\n",
       "      <td>def solution():\\n    # Naruto starts with 25 s...</td>\n",
       "      <td>7</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Naruto has been asked to deliver scr...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3366</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>Superman can bench press 3000 pounds.  He can ...</td>\n",
       "      <td>def solution():\\n    #Superman can bench press...</td>\n",
       "      <td>10464.96</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Superman can bench press 3000 pounds...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2036</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>Wonder Woman is in a 10-day battle. \\n\\nShe fi...</td>\n",
       "      <td>def solution():\\n    #Wonder Woman fights 30 e...</td>\n",
       "      <td>840</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Wonder Woman is in a 10-day battle. ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4426</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>12 unicorns ate 100 apples. 24 unicorns ate 20...</td>\n",
       "      <td>def solution():\\n    #12 unicorns ate 100 appl...</td>\n",
       "      <td>6000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: 12 unicorns ate 100 apples. 24 unico...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>mammoth</td>\n",
       "      <td>LeBron James has 4 MVP awards, 3 NBA Finals MV...</td>\n",
       "      <td>def solution():\\n    #LeBron James has 4 MVP a...</td>\n",
       "      <td>9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: LeBron James has 4 MVP awards, 3 NBA...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3092</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>124 cars are in a parking lot. 102 of the cars...</td>\n",
       "      <td>def solution():\\n    #Total cars\\n    total_ca...</td>\n",
       "      <td>7</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: 124 cars are in a parking lot. 102 o...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3772</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>Sally baked 240 cookies. She put them in 20 ba...</td>\n",
       "      <td>def solution():\\n    #Sally baked 240 cookies\\...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Sally baked 240 cookies. She put the...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>860</th>\n",
       "      <td>mathwell</td>\n",
       "      <td>There are 200 unicorns in a forest. 100 of the...</td>\n",
       "      <td>def solution():\\n    #Total unicorns\\n    tota...</td>\n",
       "      <td>50.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: There are 200 unicorns in a forest. ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4734 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                model                                           question  \\\n",
       "2307  finetuned_llama  The Varsity Lacrosse team has 24 players. The ...   \n",
       "354             llema  If the Super Mario can jump over a total of 5 ...   \n",
       "1397             gpt4  Naruto has been asked to deliver scrolls to hi...   \n",
       "3366  finetuned_llama  Superman can bench press 3000 pounds.  He can ...   \n",
       "2036  finetuned_llama  Wonder Woman is in a 10-day battle. \\n\\nShe fi...   \n",
       "...               ...                                                ...   \n",
       "4426  finetuned_llama  12 unicorns ate 100 apples. 24 unicorns ate 20...   \n",
       "466           mammoth  LeBron James has 4 MVP awards, 3 NBA Finals MV...   \n",
       "3092  finetuned_llama  124 cars are in a parking lot. 102 of the cars...   \n",
       "3772  finetuned_llama  Sally baked 240 cookies. She put them in 20 ba...   \n",
       "860          mathwell  There are 200 unicorns in a forest. 100 of the...   \n",
       "\n",
       "                                               solution    answer  accuracy  \\\n",
       "2307  def solution():\\n    #The Varsity Lacrosse tea...        30       NaN   \n",
       "354   def solution():\\n    #The super Mario can jump...       2.5       NaN   \n",
       "1397  def solution():\\n    # Naruto starts with 25 s...         7       1.0   \n",
       "3366  def solution():\\n    #Superman can bench press...  10464.96       1.0   \n",
       "2036  def solution():\\n    #Wonder Woman fights 30 e...       840       NaN   \n",
       "...                                                 ...       ...       ...   \n",
       "4426  def solution():\\n    #12 unicorns ate 100 appl...      6000       0.0   \n",
       "466   def solution():\\n    #LeBron James has 4 MVP a...         9       1.0   \n",
       "3092  def solution():\\n    #Total cars\\n    total_ca...         7       1.0   \n",
       "3772  def solution():\\n    #Sally baked 240 cookies\\...      12.0       1.0   \n",
       "860   def solution():\\n    #Total unicorns\\n    tota...      50.0       1.0   \n",
       "\n",
       "      solvability  appropriateness  mac  \\\n",
       "2307            0              0.0    0   \n",
       "354             0              NaN    0   \n",
       "1397            1              1.0    1   \n",
       "3366            1              1.0    1   \n",
       "2036            0              0.0    0   \n",
       "...           ...              ...  ...   \n",
       "4426            1              1.0    0   \n",
       "466             1              1.0    1   \n",
       "3092            1              1.0    1   \n",
       "3772            1              1.0    1   \n",
       "860             1              1.0    1   \n",
       "\n",
       "                                                 prompt  \\\n",
       "2307  Below is an instruction that describes a task....   \n",
       "354   Below is an instruction that describes a task....   \n",
       "1397  Below is an instruction that describes a task....   \n",
       "3366  Below is an instruction that describes a task....   \n",
       "2036  Below is an instruction that describes a task....   \n",
       "...                                                 ...   \n",
       "4426  Below is an instruction that describes a task....   \n",
       "466   Below is an instruction that describes a task....   \n",
       "3092  Below is an instruction that describes a task....   \n",
       "3772  Below is an instruction that describes a task....   \n",
       "860   Below is an instruction that describes a task....   \n",
       "\n",
       "                                             completion  label  \n",
       "2307  Question: The Varsity Lacrosse team has 24 pla...  False  \n",
       "354   Question: If the Super Mario can jump over a t...  False  \n",
       "1397  Question: Naruto has been asked to deliver scr...   True  \n",
       "3366  Question: Superman can bench press 3000 pounds...   True  \n",
       "2036  Question: Wonder Woman is in a 10-day battle. ...  False  \n",
       "...                                                 ...    ...  \n",
       "4426  Question: 12 unicorns ate 100 apples. 24 unico...  False  \n",
       "466   Question: LeBron James has 4 MVP awards, 3 NBA...   True  \n",
       "3092  Question: 124 cars are in a parking lot. 102 o...   True  \n",
       "3772  Question: Sally baked 240 cookies. She put the...   True  \n",
       "860   Question: There are 200 unicorns in a forest. ...   True  \n",
       "\n",
       "[4734 rows x 11 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotations = annotations.sample(frac = 1, random_state=42)\n",
    "annotations['prompt'] = formatted_prompt\n",
    "annotations['completion'] = \"Question: \" + annotations['question'] + \"\\nSolution:\\n\" + annotations['solution']\n",
    "def label(df):\n",
    "    if df['mac'] == 1:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "annotations['label'] = annotations.apply(label, axis =1)\n",
    "annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "afc1800a-8009-4c1b-8ba5-eccd03b70b07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prompt</th>\n",
       "      <th>completion</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2307</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: The Varsity Lacrosse team has 24 pla...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>354</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: If the Super Mario can jump over a t...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1397</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Naruto has been asked to deliver scr...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3366</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Superman can bench press 3000 pounds...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2036</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Wonder Woman is in a 10-day battle. ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4426</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: 12 unicorns ate 100 apples. 24 unico...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: LeBron James has 4 MVP awards, 3 NBA...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3092</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: 124 cars are in a parking lot. 102 o...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3772</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Sally baked 240 cookies. She put the...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>860</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: There are 200 unicorns in a forest. ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4732 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 prompt  \\\n",
       "2307  Below is an instruction that describes a task....   \n",
       "354   Below is an instruction that describes a task....   \n",
       "1397  Below is an instruction that describes a task....   \n",
       "3366  Below is an instruction that describes a task....   \n",
       "2036  Below is an instruction that describes a task....   \n",
       "...                                                 ...   \n",
       "4426  Below is an instruction that describes a task....   \n",
       "466   Below is an instruction that describes a task....   \n",
       "3092  Below is an instruction that describes a task....   \n",
       "3772  Below is an instruction that describes a task....   \n",
       "860   Below is an instruction that describes a task....   \n",
       "\n",
       "                                             completion  label  \n",
       "2307  Question: The Varsity Lacrosse team has 24 pla...  False  \n",
       "354   Question: If the Super Mario can jump over a t...  False  \n",
       "1397  Question: Naruto has been asked to deliver scr...   True  \n",
       "3366  Question: Superman can bench press 3000 pounds...   True  \n",
       "2036  Question: Wonder Woman is in a 10-day battle. ...  False  \n",
       "...                                                 ...    ...  \n",
       "4426  Question: 12 unicorns ate 100 apples. 24 unico...  False  \n",
       "466   Question: LeBron James has 4 MVP awards, 3 NBA...   True  \n",
       "3092  Question: 124 cars are in a parking lot. 102 o...   True  \n",
       "3772  Question: Sally baked 240 cookies. She put the...   True  \n",
       "860   Question: There are 200 unicorns in a forest. ...   True  \n",
       "\n",
       "[4732 rows x 3 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotations = annotations[['prompt', 'completion', 'label']]\n",
    "annotations = annotations.dropna()\n",
    "annotations.to_csv('data/kto.csv', index=False)\n",
    "annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "77e12a17-f61b-4a4e-bf8d-13fb9f066610",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.59"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(annotations['label'].mean()+1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "616cbad7-9b30-43c6-b043-b86287a36a0b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.41"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round((1-annotations['label'].mean())+1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "65337a90-323a-49c7-9d16-30eb0440aa5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Generating train split: 0 examples [00:00, ? examples/s]/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/site-packages/datasets/download/streaming_download_manager.py:784: FutureWarning: The 'verbose' keyword in pd.read_csv is deprecated and will be removed in a future version.\n",
      "  return pd.read_csv(xopen(filepath_or_buffer, \"rb\", download_config=download_config), **kwargs)\n",
      "Generating train split: 4734 examples [00:00, 133214.15 examples/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['prompt', 'completion', 'label'],\n",
       "        num_rows: 4734\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset, Dataset\n",
    "dataset = load_dataset('csv', data_files=\"data/kto.csv\")\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "f4ac4da1-7eb7-445a-9970-de923a8f2a14",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: A class has 120 students. 40 of them are in the 11th grade. 60 are in the 12th grade. 20 are in the 9th grade. How many students are in the 10th grade?\n",
      "Solution:\n",
      "def solution():\n",
      "    #The total number of students is 120\n",
      "    total_students = 120\n",
      "    #The number of 11th grade students is 40\n",
      "    eleventh_grade_students = 40\n",
      "    #The number of 12th grade students is 60\n",
      "    twelfth_grade_students = 60\n",
      "    #The number of 9th grade students is 20\n",
      "    ninth_grade_students = 20\n",
      "    #The number of 10th grade students is\n",
      "    tenth_grade_students = total_students - eleventh_grade_students - twelfth_grade_students - ninth_grade_students\n",
      "    return tenth_grade_students\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('data/mathwell.csv')\n",
    "print(df.iloc[1]['output'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "69834cdb-81bd-4baf-ae34-ed86c2f890c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: Iron Man has 3000 nanobots. He uses 1000 nanobots to heal his wounds. He then uses 2000 nanobots to rebuild his armor. How many nanobots does he have left?\n",
      "Solution:\n",
      "def solution():\n",
      "    #Iron Man started with 3000 nanobots\n",
      "    nanobots_initial = 3000\n",
      "    #He used 1000 to heal his wounds\n",
      "    nanobots_heal = 1000\n",
      "    #He used 2000 to rebuild his armor\n",
      "    nanobots_armor = 2000\n",
      "    #The answer is\n",
      "    result = nanobots_initial - nanobots_heal - nanobots_armor\n",
      "    return result\n"
     ]
    }
   ],
   "source": [
    "print(annotations.iloc[1]['completion'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "253f0c4b-d5f3-4ef9-afe9-170f77aff1cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/site-packages/trl/trainer/kto_trainer.py:417: UserWarning: When using DPODataCollatorWithPadding, you should set `max_length` in the KTOTrainer's init it will be set to `512` by default, but you should do it yourself in the future.\n",
      "  warnings.warn(\n",
      "/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/site-packages/trl/trainer/kto_trainer.py:427: UserWarning: When using DPODataCollatorWithPadding, you should set `max_prompt_length` in the KTOTrainer's init it will be set to `128` by default, but you should do it yourself in the future.\n",
      "  warnings.warn(\n",
      "/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/site-packages/trl/trainer/kto_trainer.py:457: UserWarning: When using DPODataCollatorWithPadding, you should set `remove_unused_columns=False` in your KTOConfig we have set it for you, but you should do it yourself in the future.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding special tokens.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Tokenizing train dataset: 100%|████| 4022/4022 [00:00<00:00, 5138.33 examples/s]\n",
      "Extracting KL train dataset: 100%|█| 4022/4022 [00:00<00:00, 9668.73 examples/s]\n",
      "Processing tokenized train dataset: 100%|█| 4022/4022 [00:00<00:00, 5581.83 exam\n",
      "Processing tokenized train KL dataset: 100%|█| 4022/4022 [00:00<00:00, 5936.50 e\n",
      "Tokenizing eval dataset: 100%|███████| 710/710 [00:00<00:00, 6833.97 examples/s]\n",
      "Extracting eval KL dataset: 100%|████| 710/710 [00:00<00:00, 9051.07 examples/s]\n",
      "Processing tokenized eval dataset: 100%|█| 710/710 [00:00<00:00, 5331.41 example\n",
      "Processing tokenized eval KL dataset: 100%|█| 710/710 [00:00<00:00, 6114.21 exam\n",
      "Filtering desirable examples: 100%|█| 4022/4022 [00:01<00:00, 2876.53 examples/s\n",
      "Filtering undesirable examples: 100%|█| 4022/4022 [00:01<00:00, 2934.49 examples\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "ERROR:root:code for hash blake2b was not found.\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/hashlib.py\", line 307, in <module>\n",
      "    globals()[__func_name] = __get_hash(__func_name)\n",
      "                             ^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/hashlib.py\", line 129, in __get_openssl_constructor\n",
      "    return __get_builtin_constructor(name)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/hashlib.py\", line 123, in __get_builtin_constructor\n",
      "    raise ValueError('unsupported hash type ' + name)\n",
      "ValueError: unsupported hash type blake2b\n",
      "ERROR:root:code for hash blake2s was not found.\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/hashlib.py\", line 307, in <module>\n",
      "    globals()[__func_name] = __get_hash(__func_name)\n",
      "                             ^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/hashlib.py\", line 129, in __get_openssl_constructor\n",
      "    return __get_builtin_constructor(name)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/hashlib.py\", line 123, in __get_builtin_constructor\n",
      "    raise ValueError('unsupported hash type ' + name)\n",
      "ValueError: unsupported hash type blake2s\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit:\n"
     ]
    },
    {
     "ename": "Abort",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAbort\u001b[0m                                     Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[43], line 89\u001b[0m\n\u001b[1;32m     65\u001b[0m training_args \u001b[38;5;241m=\u001b[39m KTOConfig(\n\u001b[1;32m     66\u001b[0m     beta\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.1\u001b[39m,\n\u001b[1;32m     67\u001b[0m     desirable_weight\u001b[38;5;241m=\u001b[39mpos_label,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     77\u001b[0m     report_to \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwandb\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m     78\u001b[0m )\n\u001b[1;32m     80\u001b[0m kto_trainer \u001b[38;5;241m=\u001b[39m KTOTrainer(\n\u001b[1;32m     81\u001b[0m     model,\n\u001b[1;32m     82\u001b[0m     model_ref,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     86\u001b[0m     tokenizer\u001b[38;5;241m=\u001b[39mtokenizer,\n\u001b[1;32m     87\u001b[0m )\n\u001b[0;32m---> 89\u001b[0m \u001b[43mkto_trainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/transformers/trainer.py:1859\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1857\u001b[0m         hf_hub_utils\u001b[38;5;241m.\u001b[39menable_progress_bars()\n\u001b[1;32m   1858\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1859\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1860\u001b[0m \u001b[43m        \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1861\u001b[0m \u001b[43m        \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1862\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1863\u001b[0m \u001b[43m        \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1864\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/transformers/trainer.py:2116\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   2113\u001b[0m model\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[1;32m   2114\u001b[0m grad_norm: Optional[\u001b[38;5;28mfloat\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m-> 2116\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcallback_handler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mon_train_begin\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcontrol\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2118\u001b[0m \u001b[38;5;66;03m# Skip the first epochs_trained epochs to get the random state of the dataloader at the right point.\u001b[39;00m\n\u001b[1;32m   2119\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m args\u001b[38;5;241m.\u001b[39mignore_data_skip:\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/transformers/trainer_callback.py:371\u001b[0m, in \u001b[0;36mCallbackHandler.on_train_begin\u001b[0;34m(self, args, state, control)\u001b[0m\n\u001b[1;32m    369\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mon_train_begin\u001b[39m(\u001b[38;5;28mself\u001b[39m, args: TrainingArguments, state: TrainerState, control: TrainerControl):\n\u001b[1;32m    370\u001b[0m     control\u001b[38;5;241m.\u001b[39mshould_training_stop \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m--> 371\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_event\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mon_train_begin\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontrol\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/transformers/trainer_callback.py:415\u001b[0m, in \u001b[0;36mCallbackHandler.call_event\u001b[0;34m(self, event, args, state, control, **kwargs)\u001b[0m\n\u001b[1;32m    413\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcall_event\u001b[39m(\u001b[38;5;28mself\u001b[39m, event, args, state, control, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m    414\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m callback \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallbacks:\n\u001b[0;32m--> 415\u001b[0m         result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mgetattr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mcallback\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevent\u001b[49m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    416\u001b[0m \u001b[43m            \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    417\u001b[0m \u001b[43m            \u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    418\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcontrol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    419\u001b[0m \u001b[43m            \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    420\u001b[0m \u001b[43m            \u001b[49m\u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtokenizer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    421\u001b[0m \u001b[43m            \u001b[49m\u001b[43moptimizer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptimizer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    422\u001b[0m \u001b[43m            \u001b[49m\u001b[43mlr_scheduler\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlr_scheduler\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    423\u001b[0m \u001b[43m            \u001b[49m\u001b[43mtrain_dataloader\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_dataloader\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    424\u001b[0m \u001b[43m            \u001b[49m\u001b[43meval_dataloader\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meval_dataloader\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    425\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    426\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    427\u001b[0m         \u001b[38;5;66;03m# A Callback can skip the return of `control` if it doesn't change it.\u001b[39;00m\n\u001b[1;32m    428\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/transformers/integrations/integration_utils.py:768\u001b[0m, in \u001b[0;36mWandbCallback.on_train_begin\u001b[0;34m(self, args, state, control, model, **kwargs)\u001b[0m\n\u001b[1;32m    766\u001b[0m     args\u001b[38;5;241m.\u001b[39mrun_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    767\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_initialized:\n\u001b[0;32m--> 768\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msetup\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/transformers/integrations/integration_utils.py:741\u001b[0m, in \u001b[0;36mWandbCallback.setup\u001b[0;34m(self, args, state, model, **kwargs)\u001b[0m\n\u001b[1;32m    738\u001b[0m         init_args[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m args\u001b[38;5;241m.\u001b[39mrun_name\n\u001b[1;32m    740\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_wandb\u001b[38;5;241m.\u001b[39mrun \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 741\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_wandb\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minit\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    742\u001b[0m \u001b[43m        \u001b[49m\u001b[43mproject\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetenv\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mWANDB_PROJECT\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mhuggingface\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    743\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minit_args\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    744\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    745\u001b[0m \u001b[38;5;66;03m# add config parameters (run may have been created manually)\u001b[39;00m\n\u001b[1;32m    746\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_wandb\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mupdate(combined_dict, allow_val_change\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/wandb/sdk/wandb_init.py:1178\u001b[0m, in \u001b[0;36minit\u001b[0;34m(job_type, dir, config, project, entity, reinit, tags, group, name, notes, magic, config_exclude_keys, config_include_keys, anonymous, mode, allow_val_change, resume, force, tensorboard, sync_tensorboard, monitor_gym, save_code, id, fork_from, settings)\u001b[0m\n\u001b[1;32m   1174\u001b[0m     logger\u001b[38;5;241m.\u001b[39mexception(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124merror in wandb.init()\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39me)\n\u001b[1;32m   1176\u001b[0m \u001b[38;5;66;03m# Need to build delay into this sentry capture because our exit hooks\u001b[39;00m\n\u001b[1;32m   1177\u001b[0m \u001b[38;5;66;03m# mess with sentry's ability to send out errors before the program ends.\u001b[39;00m\n\u001b[0;32m-> 1178\u001b[0m \u001b[43mwandb\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sentry\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreraise\u001b[49m\u001b[43m(\u001b[49m\u001b[43me\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1179\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAssertionError\u001b[39;00m()\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/wandb/analytics/sentry.py:155\u001b[0m, in \u001b[0;36mSentry.reraise\u001b[0;34m(self, exc)\u001b[0m\n\u001b[1;32m    152\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexception(exc)\n\u001b[1;32m    153\u001b[0m \u001b[38;5;66;03m# this will messily add this \"reraise\" function to the stack trace,\u001b[39;00m\n\u001b[1;32m    154\u001b[0m \u001b[38;5;66;03m# but hopefully it's not too bad\u001b[39;00m\n\u001b[0;32m--> 155\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc\u001b[38;5;241m.\u001b[39mwith_traceback(sys\u001b[38;5;241m.\u001b[39mexc_info()[\u001b[38;5;241m2\u001b[39m])\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/wandb/sdk/wandb_init.py:1163\u001b[0m, in \u001b[0;36minit\u001b[0;34m(job_type, dir, config, project, entity, reinit, tags, group, name, notes, magic, config_exclude_keys, config_include_keys, anonymous, mode, allow_val_change, resume, force, tensorboard, sync_tensorboard, monitor_gym, save_code, id, fork_from, settings)\u001b[0m\n\u001b[1;32m   1161\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1162\u001b[0m     wi \u001b[38;5;241m=\u001b[39m _WandbInit()\n\u001b[0;32m-> 1163\u001b[0m     \u001b[43mwi\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msetup\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1164\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m wi\u001b[38;5;241m.\u001b[39minit()\n\u001b[1;32m   1166\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/wandb/sdk/wandb_init.py:300\u001b[0m, in \u001b[0;36m_WandbInit.setup\u001b[0;34m(self, kwargs)\u001b[0m\n\u001b[1;32m    297\u001b[0m     settings\u001b[38;5;241m.\u001b[39mupdate(init_settings, source\u001b[38;5;241m=\u001b[39mSource\u001b[38;5;241m.\u001b[39mINIT)\n\u001b[1;32m    299\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m settings\u001b[38;5;241m.\u001b[39m_offline \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m settings\u001b[38;5;241m.\u001b[39m_noop:\n\u001b[0;32m--> 300\u001b[0m     \u001b[43mwandb_login\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_login\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    301\u001b[0m \u001b[43m        \u001b[49m\u001b[43manonymous\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpop\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43manonymous\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    302\u001b[0m \u001b[43m        \u001b[49m\u001b[43mforce\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpop\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mforce\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    303\u001b[0m \u001b[43m        \u001b[49m\u001b[43m_disable_warning\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    304\u001b[0m \u001b[43m        \u001b[49m\u001b[43m_silent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msettings\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquiet\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43msettings\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msilent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    305\u001b[0m \u001b[43m        \u001b[49m\u001b[43m_entity\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mentity\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43msettings\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mentity\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    306\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    308\u001b[0m \u001b[38;5;66;03m# apply updated global state after login was handled\u001b[39;00m\n\u001b[1;32m    309\u001b[0m wl \u001b[38;5;241m=\u001b[39m wandb\u001b[38;5;241m.\u001b[39msetup()\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/wandb/sdk/wandb_login.py:334\u001b[0m, in \u001b[0;36m_login\u001b[0;34m(anonymous, key, relogin, host, force, timeout, _backend, _silent, _disable_warning, _entity)\u001b[0m\n\u001b[1;32m    331\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m logged_in\n\u001b[1;32m    333\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m key:\n\u001b[0;32m--> 334\u001b[0m     \u001b[43mwlogin\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprompt_api_key\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    336\u001b[0m \u001b[38;5;66;03m# make sure login credentials get to the backend\u001b[39;00m\n\u001b[1;32m    337\u001b[0m wlogin\u001b[38;5;241m.\u001b[39mpropogate_login()\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/wandb/sdk/wandb_login.py:256\u001b[0m, in \u001b[0;36m_WandbLogin.prompt_api_key\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    255\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mprompt_api_key\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 256\u001b[0m     key, status \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_prompt_api_key\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    257\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m==\u001b[39m ApiKeyStatus\u001b[38;5;241m.\u001b[39mNOTTY:\n\u001b[1;32m    258\u001b[0m         directive \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m    259\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwandb login [your_api_key]\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    260\u001b[0m             \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_settings\u001b[38;5;241m.\u001b[39m_cli_only_mode\n\u001b[1;32m    261\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwandb.login(key=[your_api_key])\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    262\u001b[0m         )\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/wandb/sdk/wandb_login.py:236\u001b[0m, in \u001b[0;36m_WandbLogin._prompt_api_key\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    234\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m    235\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 236\u001b[0m         key \u001b[38;5;241m=\u001b[39m \u001b[43mapikey\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprompt_api_key\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    237\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_settings\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    238\u001b[0m \u001b[43m            \u001b[49m\u001b[43mapi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mapi\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    239\u001b[0m \u001b[43m            \u001b[49m\u001b[43mno_offline\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_settings\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforce\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_settings\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    240\u001b[0m \u001b[43m            \u001b[49m\u001b[43mno_create\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_settings\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforce\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_settings\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    241\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    242\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    243\u001b[0m         \u001b[38;5;66;03m# invalid key provided, try again\u001b[39;00m\n\u001b[1;32m    244\u001b[0m         wandb\u001b[38;5;241m.\u001b[39mtermerror(e\u001b[38;5;241m.\u001b[39margs[\u001b[38;5;241m0\u001b[39m])\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/wandb/sdk/lib/apikey.py:151\u001b[0m, in \u001b[0;36mprompt_api_key\u001b[0;34m(settings, api, input_callback, browser_callback, no_offline, no_create, local)\u001b[0m\n\u001b[1;32m    145\u001b[0m         wandb\u001b[38;5;241m.\u001b[39mtermlog(\n\u001b[1;32m    146\u001b[0m             \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLogging into \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mhost\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. (Learn how to deploy a W&B server locally: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mwburls\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwandb_server\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    147\u001b[0m         )\n\u001b[1;32m    148\u001b[0m     wandb\u001b[38;5;241m.\u001b[39mtermlog(\n\u001b[1;32m    149\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou can find your API key in your browser here: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mapp_url\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/authorize\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    150\u001b[0m     )\n\u001b[0;32m--> 151\u001b[0m     key \u001b[38;5;241m=\u001b[39m \u001b[43minput_callback\u001b[49m\u001b[43m(\u001b[49m\u001b[43mapi_ask\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mstrip()\n\u001b[1;32m    152\u001b[0m write_key(settings, key, api\u001b[38;5;241m=\u001b[39mapi)\n\u001b[1;32m    153\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m key  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/click/termui.py:164\u001b[0m, in \u001b[0;36mprompt\u001b[0;34m(text, default, hide_input, confirmation_prompt, type, value_proc, prompt_suffix, show_default, err, show_choices)\u001b[0m\n\u001b[1;32m    162\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m    163\u001b[0m     \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 164\u001b[0m         value \u001b[38;5;241m=\u001b[39m \u001b[43mprompt_func\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    165\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m value:\n\u001b[1;32m    166\u001b[0m             \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.11.5/lib/python3.11/site-packages/click/termui.py:147\u001b[0m, in \u001b[0;36mprompt.<locals>.prompt_func\u001b[0;34m(text)\u001b[0m\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m hide_input:\n\u001b[1;32m    146\u001b[0m     echo(\u001b[38;5;28;01mNone\u001b[39;00m, err\u001b[38;5;241m=\u001b[39merr)\n\u001b[0;32m--> 147\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m Abort() \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "\u001b[0;31mAbort\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset, Dataset\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "import transformers\n",
    "from trl import KTOConfig, KTOTrainer, ModelConfig, get_peft_config, setup_chat_format\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "from typing import Optional, Dict, Sequence\n",
    "df = pd.read_csv('data/kto.csv')\n",
    "neg_label = round(df['label'].mean()+1, 2)\n",
    "pos_label = round((1-df['label'].mean())+1, 2)\n",
    "dataset = load_dataset('csv', data_files=\"data/kto.csv\")\n",
    "dataset = dataset['train'].train_test_split(test_size = .15, seed = 42)\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"openai-community/gpt2\")\n",
    "model = AutoModelForCausalLM.from_pretrained(\"openai-community/gpt2\")\n",
    "model_ref = AutoModelForCausalLM.from_pretrained(\"openai-community/gpt2\")\n",
    "# Set special tokens\n",
    "DEFAULT_PAD_TOKEN = \"[PAD]\"\n",
    "def smart_tokenizer_and_embedding_resize(\n",
    "    special_tokens_dict: Dict,\n",
    "    tokenizer: transformers.PreTrainedTokenizer,\n",
    "    model: transformers.PreTrainedModel, model2: transformers.PreTrainedModel,\n",
    "):\n",
    "    \"\"\"Resize tokenizer and embedding.\n",
    "\n",
    "    Note: This is the unoptimized version that may make your embedding size not be divisible by 64.\n",
    "    \"\"\"\n",
    "    num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)\n",
    "    model.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=64)\n",
    "    model2.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=64)\n",
    "    if num_new_tokens > 0:\n",
    "        input_embeddings = model.get_input_embeddings().weight.data\n",
    "        output_embeddings = model.get_output_embeddings().weight.data\n",
    "\n",
    "        input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)\n",
    "        output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)\n",
    "\n",
    "        input_embeddings[-num_new_tokens:] = input_embeddings_avg\n",
    "        output_embeddings[-num_new_tokens:] = output_embeddings_avg\n",
    "\n",
    "        input_embeddings = model2.get_input_embeddings().weight.data\n",
    "        output_embeddings = model2.get_output_embeddings().weight.data\n",
    "\n",
    "        input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)\n",
    "        output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)\n",
    "\n",
    "        input_embeddings[-num_new_tokens:] = input_embeddings_avg\n",
    "        output_embeddings[-num_new_tokens:] = output_embeddings_avg\n",
    "        \n",
    "if tokenizer._pad_token is None:\n",
    "    smart_tokenizer_and_embedding_resize(\n",
    "        special_tokens_dict=dict(pad_token=DEFAULT_PAD_TOKEN),\n",
    "        tokenizer=tokenizer,\n",
    "        model=model, model2 = model_ref, \n",
    "    )\n",
    "\n",
    "print('Adding special tokens.')\n",
    "tokenizer.add_special_tokens({\n",
    "        \"eos_token\": tokenizer.convert_ids_to_tokens(model.config.eos_token_id),\n",
    "        \"bos_token\": tokenizer.convert_ids_to_tokens(model.config.bos_token_id),\n",
    "#                 \"unk_token\": tokenizer.convert_ids_to_tokens(\n",
    "#                     model.config.pad_token_id if model.config.pad_token_id != -1 else tokenizer.pad_token_id\n",
    "#                 ),\n",
    "})\n",
    "training_args = KTOConfig(\n",
    "    beta=0.1,\n",
    "    desirable_weight=pos_label,\n",
    "    undesirable_weight=neg_label,\n",
    "    warmup_ratio = 0.1,\n",
    "    num_train_epochs = 3,\n",
    "    learning_rate = 1e-4, \n",
    "    per_device_train_batch_size = 8,\n",
    "    output_dir=\"./mathwell8b-kto\", \n",
    "    save_steps = 250, \n",
    "    do_eval = True,\n",
    "    eval_steps = 500,\n",
    "    report_to = 'wandb',\n",
    ")\n",
    "\n",
    "kto_trainer = KTOTrainer(\n",
    "    model,\n",
    "    model_ref,\n",
    "    args=training_args,\n",
    "    train_dataset=dataset['train'],\n",
    "    eval_dataset = dataset['test'],\n",
    "    tokenizer=tokenizer,\n",
    ")\n",
    "\n",
    "kto_trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "933b58c5-4959-4f30-8327-88cb875ba9cf",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "can only concatenate str (not \"NoneType\") to str",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[27], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m prompt_and_completion \u001b[38;5;241m=\u001b[39m \u001b[43m[\u001b[49m\n\u001b[1;32m      2\u001b[0m \u001b[43m    \u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mcompletion\u001b[49m\n\u001b[1;32m      3\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcompletion\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mzip\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtrain\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mprompt\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtrain\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcompletion\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mand\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mcompletion\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\n\u001b[1;32m      5\u001b[0m \u001b[43m]\u001b[49m\n",
      "Cell \u001b[0;32mIn[27], line 2\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      1\u001b[0m prompt_and_completion \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m----> 2\u001b[0m     \u001b[43mprompt\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mcompletion\u001b[49m\n\u001b[1;32m      3\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m prompt, completion \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(dataset[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprompt\u001b[39m\u001b[38;5;124m\"\u001b[39m], dataset[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcompletion\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n\u001b[1;32m      4\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m prompt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m completion \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m      5\u001b[0m ]\n",
      "\u001b[0;31mTypeError\u001b[0m: can only concatenate str (not \"NoneType\") to str"
     ]
    }
   ],
   "source": [
    "prompt_and_completion = [\n",
    "    prompt + completion\n",
    "    for prompt, completion in zip(dataset['train'][\"prompt\"], dataset['train'][\"completion\"])\n",
    "    if prompt is not None and completion is None\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "3b6fb5dd-4fbf-462b-b295-f9361ae85eff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prompt</th>\n",
       "      <th>completion</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Patrick Mahomes has 4000 passing yar...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: There are 1000 students at a school....</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: A small pizzeria sells 8 slices of p...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: A candy store sells 500 candy canes....</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Wonder Woman has 100 lassos. She las...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4729</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: There are 20 planes in the sky. 5 ar...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4730</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: A pizza is cut into 8 slices. 2 of t...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4731</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: 5000 people live in a small town. 20...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4732</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Spiderman can shoot 3 webs per secon...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4733</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: The Teenage Mutant Ninja Turtles are...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4734 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 prompt  \\\n",
       "0     Below is an instruction that describes a task....   \n",
       "1     Below is an instruction that describes a task....   \n",
       "2     Below is an instruction that describes a task....   \n",
       "3     Below is an instruction that describes a task....   \n",
       "4     Below is an instruction that describes a task....   \n",
       "...                                                 ...   \n",
       "4729  Below is an instruction that describes a task....   \n",
       "4730  Below is an instruction that describes a task....   \n",
       "4731  Below is an instruction that describes a task....   \n",
       "4732  Below is an instruction that describes a task....   \n",
       "4733  Below is an instruction that describes a task....   \n",
       "\n",
       "                                             completion  label  \n",
       "0     Question: Patrick Mahomes has 4000 passing yar...  False  \n",
       "1     Question: There are 1000 students at a school....   True  \n",
       "2     Question: A small pizzeria sells 8 slices of p...  False  \n",
       "3     Question: A candy store sells 500 candy canes....  False  \n",
       "4     Question: Wonder Woman has 100 lassos. She las...  False  \n",
       "...                                                 ...    ...  \n",
       "4729  Question: There are 20 planes in the sky. 5 ar...   True  \n",
       "4730  Question: A pizza is cut into 8 slices. 2 of t...   True  \n",
       "4731  Question: 5000 people live in a small town. 20...  False  \n",
       "4732  Question: Spiderman can shoot 3 webs per secon...   True  \n",
       "4733  Question: The Teenage Mutant Ninja Turtles are...   True  \n",
       "\n",
       "[4734 rows x 3 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/kto.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "e44b5bab-156f-4994-9344-9378f1550a9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prompt</th>\n",
       "      <th>completion</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Patrick Mahomes has 4000 passing yar...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: There are 1000 students at a school....</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: A small pizzeria sells 8 slices of p...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: A candy store sells 500 candy canes....</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Wonder Woman has 100 lassos. She las...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4729</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: There are 20 planes in the sky. 5 ar...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4730</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: A pizza is cut into 8 slices. 2 of t...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4731</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: 5000 people live in a small town. 20...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4732</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: Spiderman can shoot 3 webs per secon...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4733</th>\n",
       "      <td>Below is an instruction that describes a task....</td>\n",
       "      <td>Question: The Teenage Mutant Ninja Turtles are...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4732 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 prompt  \\\n",
       "0     Below is an instruction that describes a task....   \n",
       "1     Below is an instruction that describes a task....   \n",
       "2     Below is an instruction that describes a task....   \n",
       "3     Below is an instruction that describes a task....   \n",
       "4     Below is an instruction that describes a task....   \n",
       "...                                                 ...   \n",
       "4729  Below is an instruction that describes a task....   \n",
       "4730  Below is an instruction that describes a task....   \n",
       "4731  Below is an instruction that describes a task....   \n",
       "4732  Below is an instruction that describes a task....   \n",
       "4733  Below is an instruction that describes a task....   \n",
       "\n",
       "                                             completion  label  \n",
       "0     Question: Patrick Mahomes has 4000 passing yar...  False  \n",
       "1     Question: There are 1000 students at a school....   True  \n",
       "2     Question: A small pizzeria sells 8 slices of p...  False  \n",
       "3     Question: A candy store sells 500 candy canes....  False  \n",
       "4     Question: Wonder Woman has 100 lassos. She las...  False  \n",
       "...                                                 ...    ...  \n",
       "4729  Question: There are 20 planes in the sky. 5 ar...   True  \n",
       "4730  Question: A pizza is cut into 8 slices. 2 of t...   True  \n",
       "4731  Question: 5000 people live in a small town. 20...  False  \n",
       "4732  Question: Spiderman can shoot 3 webs per secon...   True  \n",
       "4733  Question: The Teenage Mutant Ninja Turtles are...   True  \n",
       "\n",
       "[4732 rows x 3 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.dropna()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "26584eec-14db-401e-bba2-b3688918d782",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/kto.csv')\n",
    "neg_label = round(df['label'].mean()+1, 2)\n",
    "pos_label = round((1-df['label'].mean())+1, 2)\n",
    "dataset = load_dataset('csv', data_files=\"data/kto.csv\")\n",
    "dataset = dataset['train'].train_test_split(test_size = .05, seed = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "41465130-a23d-4f93-b828-7e1c27b138b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['prompt', 'completion', 'label'],\n",
       "        num_rows: 4497\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['prompt', 'completion', 'label'],\n",
       "        num_rows: 237\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ccc29a3d-1562-49ed-88e0-c53cb853f6f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_prompt = f\"Write a grade school math word problem about pizza and Python function with a commented out step-by-step solution to solve the word problem.\"\n",
    "final_prompt = f\"Below is an instruction that describes a task. \" + f\"Write a response that appropriately completes the request.\\n\\n\" + f\"### Instruction:\\n{final_prompt}\\n\\n### Response: \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "54040393-258a-4969-9006-25749174c8f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
      "\n",
      "### Instruction:\n",
      "Write a grade school math word problem about pizza and Python function with a commented out step-by-step solution to solve the word problem.\n",
      "\n",
      "### Response: \n"
     ]
    }
   ],
   "source": [
    "print(final_prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24a3081b-1cfc-4592-96b2-2771a26349b2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}