Parse_LlamaQA_Questions-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "17ae65ad-f11b-442a-b9b2-6b41013b7eb1",
   "metadata": {},
   "source": [
    "# Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8d610c0d-00fd-4ea8-b91d-ece6e872c721",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74a1913a-3506-405c-852d-d80d0464b85f",
   "metadata": {},
   "source": [
    "# Parse First Text File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bb591ed4-de02-4f99-a272-cae18d1c29a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Percentage of Questions with Executable Code: 62.48263084761464'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = 'llama_QA_questions.txt'\n",
    "data = []\n",
    "\n",
    "with open(file_path, 'r') as file:\n",
    "    text = file.read()\n",
    "    sections = text.split(\"Topic: \")\n",
    "    #sections = text.split(\"### Response:\")\n",
    "    del sections[0]  # Remove the initial part before the question\n",
    "    total_questions1 = len(sections)\n",
    "    for section in sections:\n",
    "        try:\n",
    "            topic = section.split(\"### Response:\")[0].strip()\n",
    "            section = section.split(\"### Response:\")[1].strip()\n",
    "            question = section.split(\"Question:\")[1].strip()\n",
    "            question = question.split(\"Solution:\")[0].strip()\n",
    "            solution = section.split(\"Solution:\")[1].strip()\n",
    "            if \"##\" in question:\n",
    "                question = question.split(\"##\")[0]\n",
    "            if \"\\nBel\" in solution:\n",
    "                solution = solution.split(\"\\nBel\")[0]\n",
    "            if \"##\" in solution:\n",
    "                solution = solution.split(\"##\")[0]\n",
    "            if \"return\" not in solution:\n",
    "                try: \n",
    "                    solution = solution + '\\n' + '    return result'\n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                            # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, 'topic': topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                                pass\n",
    "                except:\n",
    "                    pass\n",
    "            if \"return\" in solution:\n",
    "                try: \n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                        # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                            pass\n",
    "                except:\n",
    "                    pass\n",
    "        except:\n",
    "            pass\n",
    "df1 = pd.DataFrame.from_dict(data)\n",
    "result1 = \"Percentage of Questions with Executable Code: \" + str((len(df1)/total_questions1)*100)\n",
    "result1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e596ec0d-a5cd-4a70-9429-21b88c7ba48c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Percentage of Executable Code that Contains Topic: 81.98665678280209'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#this needs to be rewritten\n",
    "def check_topic(df):\n",
    "    count = 0\n",
    "    for i in range(0, len(df)):\n",
    "        if df.iloc[i]['topic'] in df.iloc[i]['question']:\n",
    "            count+=1\n",
    "            \n",
    "    return \"Percentage of Executable Code that Contains Topic: \" + str((count/len(df))*100)\n",
    "        \n",
    "    \n",
    "result2 = check_topic(df1)\n",
    "result2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8b271067-f922-4e3c-8b58-99a9e150b358",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>In 2017, Captain America was 100 years old. 20...</td>\n",
       "      <td>def solution():\\n    #Captain America was 100 ...</td>\n",
       "      <td>33</td>\n",
       "      <td>Captain America</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The volleyball team has 12 members. 5 of the m...</td>\n",
       "      <td>def solution():\\n    #The volleyball team has ...</td>\n",
       "      <td>0</td>\n",
       "      <td>volleyball</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3 of the Ninja Turtles ate 12 pizzas. 2 ate 14...</td>\n",
       "      <td>def solution():\\n    #3 turtles ate 12 pizzas\\...</td>\n",
       "      <td>82</td>\n",
       "      <td>Ninja Turtles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>T'Challa, the Black Panther, has 100 vibranium...</td>\n",
       "      <td>def solution():\\n    #T'Challa started with 10...</td>\n",
       "      <td>55</td>\n",
       "      <td>Black Panther</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The New England Patriots and the Green Bay Pac...</td>\n",
       "      <td>def solution():\\n    #The Packers are 2/3 of t...</td>\n",
       "      <td>6</td>\n",
       "      <td>football</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1344</th>\n",
       "      <td>Superman can leap 1000 feet in a single bound....</td>\n",
       "      <td>def solution():\\n    #Superman can leap 1000 f...</td>\n",
       "      <td>18000.0</td>\n",
       "      <td>Superman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1345</th>\n",
       "      <td>Wakanda, the home of the Black Panther, is a s...</td>\n",
       "      <td>def solution():\\n    #Total Wakandan soldiers\\...</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>Black Panther</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346</th>\n",
       "      <td>Batman has 112 batarangs. He uses 11 batarangs...</td>\n",
       "      <td>def solution():\\n    #Batman started with 112 ...</td>\n",
       "      <td>83</td>\n",
       "      <td>Batman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1347</th>\n",
       "      <td>The Toronto Raptors have 2000 points. The Bost...</td>\n",
       "      <td>def solution():\\n    #The Raptors have 2000 po...</td>\n",
       "      <td>3200</td>\n",
       "      <td>basketball</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1348</th>\n",
       "      <td>Spiderman, the web-slinging superhero, is chas...</td>\n",
       "      <td>def solution():\\n    #Spiderman started with 2...</td>\n",
       "      <td>16</td>\n",
       "      <td>Spiderman</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1349 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question  \\\n",
       "0     In 2017, Captain America was 100 years old. 20...   \n",
       "1     The volleyball team has 12 members. 5 of the m...   \n",
       "2     3 of the Ninja Turtles ate 12 pizzas. 2 ate 14...   \n",
       "3     T'Challa, the Black Panther, has 100 vibranium...   \n",
       "4     The New England Patriots and the Green Bay Pac...   \n",
       "...                                                 ...   \n",
       "1344  Superman can leap 1000 feet in a single bound....   \n",
       "1345  Wakanda, the home of the Black Panther, is a s...   \n",
       "1346  Batman has 112 batarangs. He uses 11 batarangs...   \n",
       "1347  The Toronto Raptors have 2000 points. The Bost...   \n",
       "1348  Spiderman, the web-slinging superhero, is chas...   \n",
       "\n",
       "                                               solution   answer  \\\n",
       "0     def solution():\\n    #Captain America was 100 ...       33   \n",
       "1     def solution():\\n    #The volleyball team has ...        0   \n",
       "2     def solution():\\n    #3 turtles ate 12 pizzas\\...       82   \n",
       "3     def solution():\\n    #T'Challa started with 10...       55   \n",
       "4     def solution():\\n    #The Packers are 2/3 of t...        6   \n",
       "...                                                 ...      ...   \n",
       "1344  def solution():\\n    #Superman can leap 1000 f...  18000.0   \n",
       "1345  def solution():\\n    #Total Wakandan soldiers\\...   1000.0   \n",
       "1346  def solution():\\n    #Batman started with 112 ...       83   \n",
       "1347  def solution():\\n    #The Raptors have 2000 po...     3200   \n",
       "1348  def solution():\\n    #Spiderman started with 2...       16   \n",
       "\n",
       "                topic  \n",
       "0     Captain America  \n",
       "1          volleyball  \n",
       "2       Ninja Turtles  \n",
       "3       Black Panther  \n",
       "4            football  \n",
       "...               ...  \n",
       "1344         Superman  \n",
       "1345    Black Panther  \n",
       "1346           Batman  \n",
       "1347       basketball  \n",
       "1348        Spiderman  \n",
       "\n",
       "[1349 rows x 4 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b3fd654-1ccf-4a9b-a998-09e9ce784989",
   "metadata": {},
   "source": [
    "# Parse Second Text File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a71cc7d1-2ac8-4556-ae4c-b67b3654524e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Percentage of Questions with Executable Code: 63.665447897623395'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = 'llama_QA_questions_v2.txt'\n",
    "data = []\n",
    "\n",
    "with open(file_path, 'r') as file:\n",
    "    text = file.read()\n",
    "    sections = text.split(\"Topic: \")\n",
    "    #sections = text.split(\"### Response:\")\n",
    "    del sections[0]  # Remove the initial part before the question\n",
    "    total_questions2 = len(sections)\n",
    "    for section in sections:\n",
    "        try:\n",
    "            topic = section.split(\"### Response:\")[0].strip()\n",
    "            section = section.split(\"### Response:\")[1].strip()\n",
    "            question = section.split(\"Question:\")[1].strip()\n",
    "            question = question.split(\"Solution:\")[0].strip()\n",
    "            solution = section.split(\"Solution:\")[1].strip()\n",
    "            if \"##\" in question:\n",
    "                question = question.split(\"##\")[0]\n",
    "            if \"\\nBel\" in solution:\n",
    "                solution = solution.split(\"\\nBel\")[0]\n",
    "            if \"##\" in solution:\n",
    "                solution = solution.split(\"##\")[0]\n",
    "            if \"return\" not in solution:\n",
    "                try: \n",
    "                    solution = solution + '\\n' + '    return result'\n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                            # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, 'topic': topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                                pass\n",
    "                except:\n",
    "                    pass\n",
    "            if \"return\" in solution:\n",
    "                try: \n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                        # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": topic})\n",
    "                        except:\n",
    "                            pass\n",
    "                except:\n",
    "                    pass\n",
    "        except:\n",
    "            pass\n",
    "df2 = pd.DataFrame.from_dict(data)\n",
    "result1 = \"Percentage of Questions with Executable Code: \" + str((len(df2)/total_questions2)*100)\n",
    "result1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f9b38439-f8e1-49d3-aac2-92be8a83ca97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Percentage of Executable Code that Contains Topic: 82.19669777458722'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result2 = check_topic(df2)\n",
    "result2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a0a6fa19-1a8a-44b6-ad5c-840bbd962b7f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Spiderman has 12 web-shooters. He uses 3 of th...</td>\n",
       "      <td>def solution():\\n    #Spiderman started with 1...</td>\n",
       "      <td>0</td>\n",
       "      <td>Spiderman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Taylor Swift is in the process of recording a ...</td>\n",
       "      <td>def solution():\\n    #Taylor Swift started wit...</td>\n",
       "      <td>26</td>\n",
       "      <td>Taylor Swift</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Captain America is in a rush to save the world...</td>\n",
       "      <td>def solution():\\n    #The first checkpoint is ...</td>\n",
       "      <td>5250.0</td>\n",
       "      <td>Captain America</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>There are 4 Ninja Turtles. 2 of them are 150 y...</td>\n",
       "      <td>def solution():\\n    #The ages of the Ninja Tu...</td>\n",
       "      <td>146.25</td>\n",
       "      <td>Ninja Turtles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>500 cars, 200 of which are convertibles, are l...</td>\n",
       "      <td>def solution():\\n    #There are 500 cars\\n    ...</td>\n",
       "      <td>430.0</td>\n",
       "      <td>cars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1388</th>\n",
       "      <td>120 cars are in a parking lot. 40 more cars en...</td>\n",
       "      <td>def solution():\\n    #There were 120 cars in t...</td>\n",
       "      <td>136.0</td>\n",
       "      <td>cars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1389</th>\n",
       "      <td>2009 was a good year for the Tesla car company...</td>\n",
       "      <td>def solution():\\n    #Tesla sold 2500 cars in ...</td>\n",
       "      <td>5500</td>\n",
       "      <td>cars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1390</th>\n",
       "      <td>The Charmander family of Pokémon has 3 evoluti...</td>\n",
       "      <td>def solution():\\n    #A Charmander is 2.5 feet...</td>\n",
       "      <td>9.375</td>\n",
       "      <td>Pokémon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1391</th>\n",
       "      <td>In the Mushroom Kingdom, there are 5000 toads....</td>\n",
       "      <td>def solution():\\n    #Total toads in Mushroom ...</td>\n",
       "      <td>1500</td>\n",
       "      <td>Super Mario</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1392</th>\n",
       "      <td>The 4:00 PM train from Penn Station to New Hav...</td>\n",
       "      <td>def solution():\\n    #Train from Penn Station ...</td>\n",
       "      <td>3.5</td>\n",
       "      <td>trains</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1393 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question  \\\n",
       "0     Spiderman has 12 web-shooters. He uses 3 of th...   \n",
       "1     Taylor Swift is in the process of recording a ...   \n",
       "2     Captain America is in a rush to save the world...   \n",
       "3     There are 4 Ninja Turtles. 2 of them are 150 y...   \n",
       "4     500 cars, 200 of which are convertibles, are l...   \n",
       "...                                                 ...   \n",
       "1388  120 cars are in a parking lot. 40 more cars en...   \n",
       "1389  2009 was a good year for the Tesla car company...   \n",
       "1390  The Charmander family of Pokémon has 3 evoluti...   \n",
       "1391  In the Mushroom Kingdom, there are 5000 toads....   \n",
       "1392  The 4:00 PM train from Penn Station to New Hav...   \n",
       "\n",
       "                                               solution  answer  \\\n",
       "0     def solution():\\n    #Spiderman started with 1...       0   \n",
       "1     def solution():\\n    #Taylor Swift started wit...      26   \n",
       "2     def solution():\\n    #The first checkpoint is ...  5250.0   \n",
       "3     def solution():\\n    #The ages of the Ninja Tu...  146.25   \n",
       "4     def solution():\\n    #There are 500 cars\\n    ...   430.0   \n",
       "...                                                 ...     ...   \n",
       "1388  def solution():\\n    #There were 120 cars in t...   136.0   \n",
       "1389  def solution():\\n    #Tesla sold 2500 cars in ...    5500   \n",
       "1390  def solution():\\n    #A Charmander is 2.5 feet...   9.375   \n",
       "1391  def solution():\\n    #Total toads in Mushroom ...    1500   \n",
       "1392  def solution():\\n    #Train from Penn Station ...     3.5   \n",
       "\n",
       "                topic  \n",
       "0           Spiderman  \n",
       "1        Taylor Swift  \n",
       "2     Captain America  \n",
       "3       Ninja Turtles  \n",
       "4                cars  \n",
       "...               ...  \n",
       "1388             cars  \n",
       "1389             cars  \n",
       "1390          Pokémon  \n",
       "1391      Super Mario  \n",
       "1392           trains  \n",
       "\n",
       "[1393 rows x 4 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "030d0b55-bc47-4b0a-a323-9ff02a41a9cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percent of all finetuned Llama questions with executable code 63.07798481711525\n",
      "Percent of all finetuned Llama questions with 0 as answer: 4.704595185995624\n",
      "2742 4347\n"
     ]
    }
   ],
   "source": [
    "print(f\"Percent of all finetuned Llama questions with executable code {(len(df1)+len(df2))/(total_questions1+total_questions2)*100}\")\n",
    "print(f\"Percent of all finetuned Llama questions with 0 as answer: {(len(df1[df1['answer']==0])+len(df2[df2['answer']==0]))/(len(df1)+len(df2))*100}\")\n",
    "print((len(df1)+len(df2)), total_questions1+total_questions2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5747cdf6-76f2-4476-857b-38a0630f9e45",
   "metadata": {},
   "source": [
    "# Parse General Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4f7129c2-5f82-47fb-a302-12d7c75f2705",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "387420489\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Percentage of Questions with Executable Code: 32.276938078985864'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = 'llama_QA_questions_general.txt'\n",
    "data = []\n",
    "\n",
    "with open(file_path, 'r') as file:\n",
    "    text = file.read()\n",
    "    #sections = text.split(\"Topic: \")\n",
    "    sections = text.split(\"### Response:\")\n",
    "    del sections[0]  # Remove the initial part before the question\n",
    "    total_questions = len(sections)\n",
    "    for section in sections:\n",
    "        try:\n",
    "            question = section.split(\"Question:\")[1].strip()\n",
    "            question = question.split(\"Solution:\")[0].strip()\n",
    "            solution = section.split(\"Solution:\")[1].strip()\n",
    "            if \"##\" in question:\n",
    "                question = question.split(\"##\")[0]\n",
    "            if \"\\n\\n\" in question:\n",
    "                question = question.split(\"\\n\\n\")\n",
    "                question = \"\\n\".join(question)\n",
    "            if \"\\nBel\" in solution:\n",
    "                solution = solution.split(\"\\nBel\")[0]\n",
    "            if \"##\" in solution:\n",
    "                solution = solution.split(\"##\")[0]\n",
    "            if \"return\" not in solution:\n",
    "                try: \n",
    "                    solution = solution + '\\n' + '    return result'\n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                            # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, 'topic': \"\"})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": \"\"})\n",
    "                        except:\n",
    "                                pass\n",
    "                except:\n",
    "                    pass\n",
    "            if \"return\" in solution:\n",
    "                try: \n",
    "                    solution_text = solution\n",
    "                    try:\n",
    "                        # Execute the Python code\n",
    "                        exec(solution)\n",
    "                        answer = solution()\n",
    "                        if answer==None:\n",
    "                            continue\n",
    "                        data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": \"\"})\n",
    "                    except: \n",
    "                        try: #pick up questions that did not indent the code block\n",
    "                            solution_new = \"\"\n",
    "                            lines = solution.split('\\n')\n",
    "                            indent = False\n",
    "                            for line in lines:\n",
    "                                if line.startswith('def solution():'):\n",
    "                                    solution_new += line + '\\n'\n",
    "\n",
    "                                if not line.startswith('def solution():') and not line.startswith('    return result'):\n",
    "                                    solution_new += '    ' + line + '\\n'\n",
    "\n",
    "                                if line.startswith('    return result'):\n",
    "                                    solution_new += line\n",
    "                            solution_text = solution_new\n",
    "                            # Execute the Python code\n",
    "                            exec(solution_new)\n",
    "                            answer = solution()\n",
    "                            if answer==None:\n",
    "                                continue\n",
    "                            data.append({\"question\": question, \"solution\": solution_text, 'answer': answer, \"topic\": \"\"})\n",
    "                        except:\n",
    "                            pass\n",
    "                except:\n",
    "                    pass\n",
    "        except:\n",
    "            pass\n",
    "df3 = pd.DataFrame.from_dict(data)\n",
    "result1 = \"Percentage of Questions with Executable Code: \" + str((len(df3)/total_questions)*100)\n",
    "result1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bae87a1d-96a1-4eae-935f-398d262fefa2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The bakers at the Beverly Hills Bakery baked 2...</td>\n",
       "      <td>def solution():\\n    #The bakers started with ...</td>\n",
       "      <td>74</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The bakers at the Beverly Hills Bakery baked 2...</td>\n",
       "      <td>def solution():\\n    #The bakers started with ...</td>\n",
       "      <td>74</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A candy store sells 1000 lollipops a day. 200 ...</td>\n",
       "      <td>def solution():\\n    #The candy store sells 10...</td>\n",
       "      <td>100</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A pig farmer has 500 pigs. 100 of the pigs are...</td>\n",
       "      <td>def solution():\\n    #There are 500 pigs\\n    ...</td>\n",
       "      <td>417.5</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The number of toys in a toy box is 20. 1/4 of ...</td>\n",
       "      <td>def solution():\\n    #The number of toys in a ...</td>\n",
       "      <td>5.0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>657</th>\n",
       "      <td>1240 toothpicks were used to make 500 aestheti...</td>\n",
       "      <td>def solution():\\n    #1240 toothpicks were use...</td>\n",
       "      <td>-58760</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>658</th>\n",
       "      <td>A bakery sells 12 baguettes, 30 croissants, an...</td>\n",
       "      <td>def solution():\\n    #baguettes, croissants, a...</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>659</th>\n",
       "      <td>A 25-gallon aquarium and a 50-gallon aquarium ...</td>\n",
       "      <td>def solution():\\n    #25-gallon aquarium is 10...</td>\n",
       "      <td>2.5</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>660</th>\n",
       "      <td>The 12-year-old is 48 inches tall. The 14-year...</td>\n",
       "      <td>def solution():\\n    #The 12-year-old is 48 in...</td>\n",
       "      <td>24</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>The 23-pound turkey was 120 degrees when it wa...</td>\n",
       "      <td>def solution():\\n    #Initial temperature of t...</td>\n",
       "      <td>-280.0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>662 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              question  \\\n",
       "0    The bakers at the Beverly Hills Bakery baked 2...   \n",
       "1    The bakers at the Beverly Hills Bakery baked 2...   \n",
       "2    A candy store sells 1000 lollipops a day. 200 ...   \n",
       "3    A pig farmer has 500 pigs. 100 of the pigs are...   \n",
       "4    The number of toys in a toy box is 20. 1/4 of ...   \n",
       "..                                                 ...   \n",
       "657  1240 toothpicks were used to make 500 aestheti...   \n",
       "658  A bakery sells 12 baguettes, 30 croissants, an...   \n",
       "659  A 25-gallon aquarium and a 50-gallon aquarium ...   \n",
       "660  The 12-year-old is 48 inches tall. The 14-year...   \n",
       "661  The 23-pound turkey was 120 degrees when it wa...   \n",
       "\n",
       "                                              solution  answer topic  \n",
       "0    def solution():\\n    #The bakers started with ...      74        \n",
       "1    def solution():\\n    #The bakers started with ...      74        \n",
       "2    def solution():\\n    #The candy store sells 10...     100        \n",
       "3    def solution():\\n    #There are 500 pigs\\n    ...   417.5        \n",
       "4    def solution():\\n    #The number of toys in a ...     5.0        \n",
       "..                                                 ...     ...   ...  \n",
       "657  def solution():\\n    #1240 toothpicks were use...  -58760        \n",
       "658  def solution():\\n    #baguettes, croissants, a...       2        \n",
       "659  def solution():\\n    #25-gallon aquarium is 10...     2.5        \n",
       "660  def solution():\\n    #The 12-year-old is 48 in...      24        \n",
       "661  def solution():\\n    #Initial temperature of t...  -280.0        \n",
       "\n",
       "[662 rows x 4 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "951f5bb4-1202-4d77-b27d-3f04f79eca11",
   "metadata": {},
   "source": [
    "# Concat DFs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b904fb0b-a6ed-41c0-ad9e-ec842540c85d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df1, df2, df3])\n",
    "df = df.drop_duplicates(subset='question')\n",
    "df = df.sample(frac = 1, random_state =42)\n",
    "question_numbers = []\n",
    "for i in range(0, len(df)):\n",
    "    question_numbers.append(f\"question_{i}\")\n",
    "df['question_number'] = question_numbers\n",
    "#df.to_csv(\"data\\llama_QA_questions.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0cdda6f2-7181-4315-8940-1a9d66b82ddf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "      <th>question_number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>A pizza is cut into 8 slices. 4 of the slices ...</td>\n",
       "      <td>def solution():\\n    #Total slices of pizza\\n ...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>pizza</td>\n",
       "      <td>question_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>There are 24 cookies in a tray. If 3/4 of the ...</td>\n",
       "      <td>def solution():\\n    #Total number of cookies\\...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>cookies</td>\n",
       "      <td>question_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>441</th>\n",
       "      <td>3250 pencils are in 500 cans. 1000 more cans a...</td>\n",
       "      <td>def solution():\\n    #Initial number of pencil...</td>\n",
       "      <td>0.0</td>\n",
       "      <td></td>\n",
       "      <td>question_2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>Barbie has 100 pink pencils. She has 25 more p...</td>\n",
       "      <td>def solution():\\n    #Barbie has 100 pink penc...</td>\n",
       "      <td>175</td>\n",
       "      <td>Barbie</td>\n",
       "      <td>question_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1280</th>\n",
       "      <td>A 747-400 can fly 10,600 miles in 14 hours. \\n...</td>\n",
       "      <td>def solution():\\n    # The 747-400 can fly 10,...</td>\n",
       "      <td>14385.714286</td>\n",
       "      <td>planes</td>\n",
       "      <td>question_4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1141</th>\n",
       "      <td>Naruto has 600 ryo. He spends 300 ryo on a new...</td>\n",
       "      <td>def solution():\\n    #Naruto started with 600 ...</td>\n",
       "      <td>0</td>\n",
       "      <td>Naruto</td>\n",
       "      <td>question_3230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1178</th>\n",
       "      <td>If 2000 cookies are baked in 100 minutes, how ...</td>\n",
       "      <td>def solution():\\n    #2000 cookies are baked i...</td>\n",
       "      <td>400.0</td>\n",
       "      <td>cookies</td>\n",
       "      <td>question_3231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>There are 4 Ninja Turtles. 2 of them are 150 y...</td>\n",
       "      <td>def solution():\\n    #The ages of the Ninja Tu...</td>\n",
       "      <td>146.25</td>\n",
       "      <td>Ninja Turtles</td>\n",
       "      <td>question_3232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>896</th>\n",
       "      <td>The Mighty Morphin Power Rangers are a superhe...</td>\n",
       "      <td>def solution():\\n    #number of Power Rangers\\...</td>\n",
       "      <td>57.5</td>\n",
       "      <td>Power Rangers</td>\n",
       "      <td>question_3233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>596</th>\n",
       "      <td>The bakery has 300 buns. The baker bakes 100 m...</td>\n",
       "      <td>def solution():\\n    #The bakery started with ...</td>\n",
       "      <td>900</td>\n",
       "      <td></td>\n",
       "      <td>question_3234</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3235 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question  \\\n",
       "43    A pizza is cut into 8 slices. 4 of the slices ...   \n",
       "493   There are 24 cookies in a tray. If 3/4 of the ...   \n",
       "441   3250 pencils are in 500 cans. 1000 more cans a...   \n",
       "333   Barbie has 100 pink pencils. She has 25 more p...   \n",
       "1280  A 747-400 can fly 10,600 miles in 14 hours. \\n...   \n",
       "...                                                 ...   \n",
       "1141  Naruto has 600 ryo. He spends 300 ryo on a new...   \n",
       "1178  If 2000 cookies are baked in 100 minutes, how ...   \n",
       "3     There are 4 Ninja Turtles. 2 of them are 150 y...   \n",
       "896   The Mighty Morphin Power Rangers are a superhe...   \n",
       "596   The bakery has 300 buns. The baker bakes 100 m...   \n",
       "\n",
       "                                               solution        answer  \\\n",
       "43    def solution():\\n    #Total slices of pizza\\n ...           6.0   \n",
       "493   def solution():\\n    #Total number of cookies\\...           6.0   \n",
       "441   def solution():\\n    #Initial number of pencil...           0.0   \n",
       "333   def solution():\\n    #Barbie has 100 pink penc...           175   \n",
       "1280  def solution():\\n    # The 747-400 can fly 10,...  14385.714286   \n",
       "...                                                 ...           ...   \n",
       "1141  def solution():\\n    #Naruto started with 600 ...             0   \n",
       "1178  def solution():\\n    #2000 cookies are baked i...         400.0   \n",
       "3     def solution():\\n    #The ages of the Ninja Tu...        146.25   \n",
       "896   def solution():\\n    #number of Power Rangers\\...          57.5   \n",
       "596   def solution():\\n    #The bakery started with ...           900   \n",
       "\n",
       "              topic question_number  \n",
       "43            pizza      question_0  \n",
       "493         cookies      question_1  \n",
       "441                      question_2  \n",
       "333          Barbie      question_3  \n",
       "1280         planes      question_4  \n",
       "...             ...             ...  \n",
       "1141         Naruto   question_3230  \n",
       "1178        cookies   question_3231  \n",
       "3     Ninja Turtles   question_3232  \n",
       "896   Power Rangers   question_3233  \n",
       "596                   question_3234  \n",
       "\n",
       "[3235 rows x 5 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a121170c-d9b6-4dfd-8e78-3cffd10a4455",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Superman can bench press 1200 pounds.  He can also bench press 200 more pounds than the Hulk.  How many pounds can the Hulk bench press?'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_temp = df[df['question'].str.contains('Superman')]\n",
    "df_temp.iloc[-1]['question']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "600777e6-285b-475d-b3fb-34e4d0cef37b",
   "metadata": {},
   "source": [
    "# Send Each Question to .txt file for Zooniverse Upload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "847988b1-54fc-4327-a5d7-520cd4cbe905",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0, len(df)):\n",
    "    question_number = df.iloc[i]['question_number']\n",
    "    question = df.iloc[i]['question']\n",
    "    solution = df.iloc[i]['solution']\n",
    "    answer = str(df.iloc[i]['answer'])\n",
    "    file_name = f\"{question_number}.txt\"\n",
    "    file_path = \"Zooniverse/\"+file_name\n",
    "    text = \"Question Number: \" + question_number + \"\\n\\nQuestion:\\n\" + question + \"\\n\\nSolution:\\n\" + solution + \"\\n\\nAnswer: \" + answer\n",
    "    with open(file_path, \"w\") as f:  # Open the file in append mode (\"a\")\n",
    "        f.write(text)  # Append the newly generated text to the file"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}