Compare_Models_Zooniverse.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "918a8dc7-cbb8-4711-8dc1-2becb1244fc8",
   "metadata": {},
   "source": [
    "## Import Dependencies and Load in Data At Correct Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "61bb9e89-2e5c-4a63-a1d5-9d7b3dec2040",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "import numpy as np\n",
    "import scipy.stats as st\n",
    "import statsmodels.api as sm \n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6fbd6228-6c2e-4db6-991a-60b5c27e0abc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>classification_id</th>\n",
       "      <th>user_name</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_ip</th>\n",
       "      <th>workflow_id</th>\n",
       "      <th>workflow_name</th>\n",
       "      <th>workflow_version</th>\n",
       "      <th>created_at</th>\n",
       "      <th>gold_standard</th>\n",
       "      <th>expert</th>\n",
       "      <th>metadata</th>\n",
       "      <th>annotations</th>\n",
       "      <th>subject_data</th>\n",
       "      <th>subject_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>501808106</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>9ccb62c8ed385fe7bc33</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>37.630</td>\n",
       "      <td>2023-07-18 16:13:14 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"9a2534a2cdf37064e69...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"90771026\":{\"retired\":null,\"Filename\":\"questi...</td>\n",
       "      <td>90771026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>501808147</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>9ccb62c8ed385fe7bc33</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>37.630</td>\n",
       "      <td>2023-07-18 16:13:27 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"9a2534a2cdf37064e69...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"90771026\":{\"retired\":null,\"Filename\":\"questi...</td>\n",
       "      <td>90771026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>501808176</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>9ccb62c8ed385fe7bc33</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>37.630</td>\n",
       "      <td>2023-07-18 16:13:36 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"9a2534a2cdf37064e69...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"90771026\":{\"retired\":null,\"Filename\":\"questi...</td>\n",
       "      <td>90771026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>501808272</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>9ccb62c8ed385fe7bc33</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>37.630</td>\n",
       "      <td>2023-07-18 16:14:12 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"9a2534a2cdf37064e69...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"90771026\":{\"retired\":null,\"Filename\":\"questi...</td>\n",
       "      <td>90771026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>501809115</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>9ccb62c8ed385fe7bc33</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>37.630</td>\n",
       "      <td>2023-07-18 16:19:32 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"9a2534a2cdf37064e69...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"90771026\":{\"retired\":null,\"Filename\":\"questi...</td>\n",
       "      <td>90771026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7284</th>\n",
       "      <td>569847408</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>2a0244cc976d3aca5b3f</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:03 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262312\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7285</th>\n",
       "      <td>569847516</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:24 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262399\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7286</th>\n",
       "      <td>569847621</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:42 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262362\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7287</th>\n",
       "      <td>569847743</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:01 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262329\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7288</th>\n",
       "      <td>569848012</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:43 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262438\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262438</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7289 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      classification_id    user_name    user_id               user_ip  \\\n",
       "0             501808106  bryanchrist  2631956.0  9ccb62c8ed385fe7bc33   \n",
       "1             501808147  bryanchrist  2631956.0  9ccb62c8ed385fe7bc33   \n",
       "2             501808176  bryanchrist  2631956.0  9ccb62c8ed385fe7bc33   \n",
       "3             501808272  bryanchrist  2631956.0  9ccb62c8ed385fe7bc33   \n",
       "4             501809115  bryanchrist  2631956.0  9ccb62c8ed385fe7bc33   \n",
       "...                 ...          ...        ...                   ...   \n",
       "7284          569847408  bryanchrist  2631956.0  2a0244cc976d3aca5b3f   \n",
       "7285          569847516  bryanchrist  2631956.0  4d1e22248ff8d5ac8526   \n",
       "7286          569847621  bryanchrist  2631956.0  4d1e22248ff8d5ac8526   \n",
       "7287          569847743  bryanchrist  2631956.0  4d1e22248ff8d5ac8526   \n",
       "7288          569848012  bryanchrist  2631956.0  4d1e22248ff8d5ac8526   \n",
       "\n",
       "      workflow_id       workflow_name  workflow_version  \\\n",
       "0           24740  Classify Questions            37.630   \n",
       "1           24740  Classify Questions            37.630   \n",
       "2           24740  Classify Questions            37.630   \n",
       "3           24740  Classify Questions            37.630   \n",
       "4           24740  Classify Questions            37.630   \n",
       "...           ...                 ...               ...   \n",
       "7284        24740  Classify Questions           142.214   \n",
       "7285        24740  Classify Questions           142.214   \n",
       "7286        24740  Classify Questions           142.214   \n",
       "7287        24740  Classify Questions           142.214   \n",
       "7288        24740  Classify Questions           142.214   \n",
       "\n",
       "                   created_at gold_standard expert  \\\n",
       "0     2023-07-18 16:13:14 UTC           NaN    NaN   \n",
       "1     2023-07-18 16:13:27 UTC           NaN    NaN   \n",
       "2     2023-07-18 16:13:36 UTC           NaN    NaN   \n",
       "3     2023-07-18 16:14:12 UTC           NaN    NaN   \n",
       "4     2023-07-18 16:19:32 UTC           NaN    NaN   \n",
       "...                       ...           ...    ...   \n",
       "7284  2024-06-27 18:26:03 UTC           NaN    NaN   \n",
       "7285  2024-06-27 18:26:24 UTC           NaN    NaN   \n",
       "7286  2024-06-27 18:26:42 UTC           NaN    NaN   \n",
       "7287  2024-06-27 18:27:01 UTC           NaN    NaN   \n",
       "7288  2024-06-27 18:27:43 UTC           NaN    NaN   \n",
       "\n",
       "                                               metadata  \\\n",
       "0     {\"source\":\"api\",\"session\":\"9a2534a2cdf37064e69...   \n",
       "1     {\"source\":\"api\",\"session\":\"9a2534a2cdf37064e69...   \n",
       "2     {\"source\":\"api\",\"session\":\"9a2534a2cdf37064e69...   \n",
       "3     {\"source\":\"api\",\"session\":\"9a2534a2cdf37064e69...   \n",
       "4     {\"source\":\"api\",\"session\":\"9a2534a2cdf37064e69...   \n",
       "...                                                 ...   \n",
       "7284  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7285  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7286  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7287  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7288  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "\n",
       "                                            annotations  \\\n",
       "0     [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "1     [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "2     [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "3     [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "4     [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "...                                                 ...   \n",
       "7284  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7285  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7286  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7287  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7288  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "\n",
       "                                           subject_data  subject_ids  \n",
       "0     {\"90771026\":{\"retired\":null,\"Filename\":\"questi...     90771026  \n",
       "1     {\"90771026\":{\"retired\":null,\"Filename\":\"questi...     90771026  \n",
       "2     {\"90771026\":{\"retired\":null,\"Filename\":\"questi...     90771026  \n",
       "3     {\"90771026\":{\"retired\":null,\"Filename\":\"questi...     90771026  \n",
       "4     {\"90771026\":{\"retired\":null,\"Filename\":\"questi...     90771026  \n",
       "...                                                 ...          ...  \n",
       "7284  {\"100262312\":{\"retired\":null,\"Filename\":\"egsm_...    100262312  \n",
       "7285  {\"100262399\":{\"retired\":null,\"Filename\":\"egsm_...    100262399  \n",
       "7286  {\"100262362\":{\"retired\":null,\"Filename\":\"egsm_...    100262362  \n",
       "7287  {\"100262329\":{\"retired\":null,\"Filename\":\"egsm_...    100262329  \n",
       "7288  {\"100262438\":{\"retired\":null,\"Filename\":\"egsm_...    100262438  \n",
       "\n",
       "[7289 rows x 14 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/model_comparison.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7a3803dc-a5b7-448e-9ff5-bd8587c61f4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>classification_id</th>\n",
       "      <th>user_name</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_ip</th>\n",
       "      <th>workflow_id</th>\n",
       "      <th>workflow_name</th>\n",
       "      <th>workflow_version</th>\n",
       "      <th>created_at</th>\n",
       "      <th>gold_standard</th>\n",
       "      <th>expert</th>\n",
       "      <th>metadata</th>\n",
       "      <th>annotations</th>\n",
       "      <th>subject_data</th>\n",
       "      <th>subject_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>518098408</td>\n",
       "      <td>jkropko</td>\n",
       "      <td>2110353.0</td>\n",
       "      <td>9bf59dbeecedb0fad009</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>109.155</td>\n",
       "      <td>2023-10-24 14:54:44 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"acac086602c98919e39...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"92652903\":{\"retired\":{\"id\":121033819,\"workfl...</td>\n",
       "      <td>92652903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>518099549</td>\n",
       "      <td>jkropko</td>\n",
       "      <td>2110353.0</td>\n",
       "      <td>207d334a2eae31af9301</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>109.155</td>\n",
       "      <td>2023-10-24 14:59:42 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"acac086602c98919e39...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"92654135\":{\"retired\":{\"id\":121038345,\"workfl...</td>\n",
       "      <td>92654135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>518100455</td>\n",
       "      <td>jkropko</td>\n",
       "      <td>2110353.0</td>\n",
       "      <td>e1e6ea4ab44bfdfebf5b</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>109.155</td>\n",
       "      <td>2023-10-24 15:03:53 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"acac086602c98919e39...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"92652729\":{\"retired\":null,\"Filename\":\"questi...</td>\n",
       "      <td>92652729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>518157865</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>7f93a3aa67d1e5d96314</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>109.155</td>\n",
       "      <td>2023-10-24 18:30:06 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"1609a40925358ed6be2...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"92653816\":{\"retired\":null,\"Filename\":\"questi...</td>\n",
       "      <td>92653816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>518158311</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>7f93a3aa67d1e5d96314</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>109.155</td>\n",
       "      <td>2023-10-24 18:31:20 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"1609a40925358ed6be2...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"92653934\":{\"retired\":{\"id\":121031492,\"workfl...</td>\n",
       "      <td>92653934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7284</th>\n",
       "      <td>569847408</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>2a0244cc976d3aca5b3f</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:03 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262312\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7285</th>\n",
       "      <td>569847516</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:24 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262399\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7286</th>\n",
       "      <td>569847621</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:42 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262362\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7287</th>\n",
       "      <td>569847743</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:01 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262329\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7288</th>\n",
       "      <td>569848012</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:43 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262438\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262438</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7271 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      classification_id    user_name    user_id               user_ip  \\\n",
       "18            518098408      jkropko  2110353.0  9bf59dbeecedb0fad009   \n",
       "19            518099549      jkropko  2110353.0  207d334a2eae31af9301   \n",
       "20            518100455      jkropko  2110353.0  e1e6ea4ab44bfdfebf5b   \n",
       "21            518157865  bryanchrist  2631956.0  7f93a3aa67d1e5d96314   \n",
       "22            518158311  bryanchrist  2631956.0  7f93a3aa67d1e5d96314   \n",
       "...                 ...          ...        ...                   ...   \n",
       "7284          569847408  bryanchrist  2631956.0  2a0244cc976d3aca5b3f   \n",
       "7285          569847516  bryanchrist  2631956.0  4d1e22248ff8d5ac8526   \n",
       "7286          569847621  bryanchrist  2631956.0  4d1e22248ff8d5ac8526   \n",
       "7287          569847743  bryanchrist  2631956.0  4d1e22248ff8d5ac8526   \n",
       "7288          569848012  bryanchrist  2631956.0  4d1e22248ff8d5ac8526   \n",
       "\n",
       "      workflow_id       workflow_name  workflow_version  \\\n",
       "18          24740  Classify Questions           109.155   \n",
       "19          24740  Classify Questions           109.155   \n",
       "20          24740  Classify Questions           109.155   \n",
       "21          24740  Classify Questions           109.155   \n",
       "22          24740  Classify Questions           109.155   \n",
       "...           ...                 ...               ...   \n",
       "7284        24740  Classify Questions           142.214   \n",
       "7285        24740  Classify Questions           142.214   \n",
       "7286        24740  Classify Questions           142.214   \n",
       "7287        24740  Classify Questions           142.214   \n",
       "7288        24740  Classify Questions           142.214   \n",
       "\n",
       "                   created_at gold_standard expert  \\\n",
       "18    2023-10-24 14:54:44 UTC           NaN    NaN   \n",
       "19    2023-10-24 14:59:42 UTC           NaN    NaN   \n",
       "20    2023-10-24 15:03:53 UTC           NaN    NaN   \n",
       "21    2023-10-24 18:30:06 UTC           NaN    NaN   \n",
       "22    2023-10-24 18:31:20 UTC           NaN    NaN   \n",
       "...                       ...           ...    ...   \n",
       "7284  2024-06-27 18:26:03 UTC           NaN    NaN   \n",
       "7285  2024-06-27 18:26:24 UTC           NaN    NaN   \n",
       "7286  2024-06-27 18:26:42 UTC           NaN    NaN   \n",
       "7287  2024-06-27 18:27:01 UTC           NaN    NaN   \n",
       "7288  2024-06-27 18:27:43 UTC           NaN    NaN   \n",
       "\n",
       "                                               metadata  \\\n",
       "18    {\"source\":\"api\",\"session\":\"acac086602c98919e39...   \n",
       "19    {\"source\":\"api\",\"session\":\"acac086602c98919e39...   \n",
       "20    {\"source\":\"api\",\"session\":\"acac086602c98919e39...   \n",
       "21    {\"source\":\"api\",\"session\":\"1609a40925358ed6be2...   \n",
       "22    {\"source\":\"api\",\"session\":\"1609a40925358ed6be2...   \n",
       "...                                                 ...   \n",
       "7284  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7285  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7286  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7287  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7288  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "\n",
       "                                            annotations  \\\n",
       "18    [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "19    [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "20    [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "21    [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "22    [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "...                                                 ...   \n",
       "7284  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7285  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7286  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7287  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7288  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "\n",
       "                                           subject_data  subject_ids  \n",
       "18    {\"92652903\":{\"retired\":{\"id\":121033819,\"workfl...     92652903  \n",
       "19    {\"92654135\":{\"retired\":{\"id\":121038345,\"workfl...     92654135  \n",
       "20    {\"92652729\":{\"retired\":null,\"Filename\":\"questi...     92652729  \n",
       "21    {\"92653816\":{\"retired\":null,\"Filename\":\"questi...     92653816  \n",
       "22    {\"92653934\":{\"retired\":{\"id\":121031492,\"workfl...     92653934  \n",
       "...                                                 ...          ...  \n",
       "7284  {\"100262312\":{\"retired\":null,\"Filename\":\"egsm_...    100262312  \n",
       "7285  {\"100262399\":{\"retired\":null,\"Filename\":\"egsm_...    100262399  \n",
       "7286  {\"100262362\":{\"retired\":null,\"Filename\":\"egsm_...    100262362  \n",
       "7287  {\"100262329\":{\"retired\":null,\"Filename\":\"egsm_...    100262329  \n",
       "7288  {\"100262438\":{\"retired\":null,\"Filename\":\"egsm_...    100262438  \n",
       "\n",
       "[7271 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(0, len(df)):\n",
    "    my_json = json.loads(df.iloc[i]['annotations'])\n",
    "    try:\n",
    "        if my_json[2]['task_label']=='Would you feel comfortable giving this question to a student? Although questions have various difficulties, a middle school student should be able to solve them all.':\n",
    "            start_index = i\n",
    "            break\n",
    "    except:\n",
    "        pass\n",
    "df = df[start_index:]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "301ef452-f6c9-4abe-a38c-618f6c1f3c9e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1850628593.py:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['started_at'] = df['metadata'].apply(extract_started_at)\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1850628593.py:16: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['started_at'] = pd.to_datetime(df['started_at'])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1850628593.py:26: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['finished_at'] = df['metadata'].apply(extract_finished_at)\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1850628593.py:27: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['finished_at'] = pd.to_datetime(df['finished_at'])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1850628593.py:30: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['time_before_submission'] = df['finished_at'] - df['started_at']\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1850628593.py:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['time_before_submission_seconds'] = df['time_before_submission'].dt.total_seconds()\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "# Function to extract 'started_at' from JSON string\n",
    "def extract_started_at(json_str):\n",
    "    try:\n",
    "        data_dict = json.loads(json_str)\n",
    "        return data_dict.get('started_at')\n",
    "    except json.JSONDecodeError:\n",
    "        return None\n",
    "\n",
    "# Apply the function to the DataFrame column\n",
    "df['started_at'] = df['metadata'].apply(extract_started_at)\n",
    "\n",
    "# Convert 'started_at' and 'created_at' to datetime\n",
    "df['started_at'] = pd.to_datetime(df['started_at'])\n",
    "\n",
    "def extract_finished_at(json_str):\n",
    "    try:\n",
    "        data_dict = json.loads(json_str)\n",
    "        return data_dict.get('finished_at')\n",
    "    except json.JSONDecodeError:\n",
    "        return None\n",
    "\n",
    "# Apply the function to the DataFrame column\n",
    "df['finished_at'] = df['metadata'].apply(extract_finished_at)\n",
    "df['finished_at'] = pd.to_datetime(df['finished_at'])\n",
    "\n",
    "# Calculate the time difference\n",
    "df['time_before_submission'] = df['finished_at'] - df['started_at']\n",
    "df['time_before_submission_seconds'] = df['time_before_submission'].dt.total_seconds()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "10be85a7-cb06-49b7-b544-94c9b8df2022",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.826 405589.721 3.4225891119974325\n"
     ]
    }
   ],
   "source": [
    "print(min(df['time_before_submission_seconds']), max(df['time_before_submission_seconds']), df['time_before_submission_seconds'].mean()/60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f3cc4141-78de-4b5e-9ff6-9b7d459369f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4282714425.py:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['time_before_submission_seconds'] = np.where(\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2wAAAIjCAYAAAB/FZhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAACStklEQVR4nOzde3zO9f/H8ee187DZbLMNc4hhzqzMOlHUhKLkFF+HhL4RRSoqh1TTgShKKoeKryipb0VJdHDMpANzDHMa5rDZ2Pnz++P6ub4u25jZ5XNdPO6323V7u96f9+f9eX0+++yy1/X+fN4fi2EYhgAAAAAATsfN7AAAAAAAAIUjYQMAAAAAJ0XCBgAAAABOioQNAAAAAJwUCRsAAAAAOCkSNgAAAABwUiRsAAAAAOCkSNgAAAAAwEmRsAEAAACAkyJhA+Aw1atXV9++fc0O45r3+uuv64YbbpC7u7uaNGlyxf1dLz+33NxcPf3004qIiJCbm5s6depkdkgOY7FYNGTIEIdvZ+/evbJYLJozZ06p922xWDRu3LhS77e4HnvsMd11112mbf9qGjdunCwWy2Wts3XrVnl4eOjvv/92UFTA9YuEDUCxzJkzRxaLRRs3bix0eatWrdSgQYMr3s63335r6h9lrub777/X008/rVtuuUWzZ8/WK6+8UqDNqlWrZLFYivVyRuf+eDz3cnNzU3h4uDp06KB169aVuN9Zs2bp9ddf14MPPqi5c+fqySefLMWoSy47O1tTp05V06ZN5e/vr4CAANWvX18DBw7Utm3bzA7vurRnzx598MEHGj16tNmhOK169eqpffv2GjNmjNmhANccD7MDAHDt2r59u9zcLu97oW+//VbTp08naSumH3/8UW5ubvrwww/l5eVVaJuoqCh9/PHHdnWjRo1SuXLl9NxzzxVoX5Kf29Xw7rvvqly5csrPz9f+/fv1/vvv6/bbb9eGDRtKNLL4448/qnLlynrzzTdLP9gr0LlzZy1dulQ9evTQgAEDlJOTo23btunrr7/WzTffrLp165odYpGqVaums2fPytPTs9T7Pnv2rDw8zPmzZerUqapRo4buuOMOU7bvKh599FG1a9dOu3fvVs2aNc0OB7hmkLABcBhvb2+zQ7hsGRkZKlu2rNlhFNvRo0fl6+tbZLImSaGhoerVq5dd3cSJExUcHFygXnLen9uDDz6o4OBg2/tOnTqpQYMGWrRoUYkStqNHjyogIKDU4svPz1d2drZ8fHxK3Mdvv/2mr7/+Wi+//HKB0Zxp06bp1KlTVxilY1ksliva/4txVL+XkpOTo3nz5unRRx81ZfuupE2bNgoMDNTcuXP14osvmh0OcM1wvq9QAVwzLrwXKicnR+PHj1dkZKR8fHwUFBSkW2+9VcuXL5ck9e3bV9OnT5ekQi/Ty8jI0IgRIxQRESFvb2/VqVNHb7zxhgzDsNvu2bNnNXToUAUHB8vPz0/33XefDh48WOAemHOX2m3dulUPPfSQAgMDdeutt0qS/vzzT/Xt21c33HCDfHx8FBYWpocffljHjx+329a5Pnbs2KFevXqpfPnyCgkJ0QsvvCDDMLR//3517NhR/v7+CgsL06RJk4p17HJzczVhwgTVrFlT3t7eql69ukaPHq2srCxbG4vFotmzZysjI8N2rErj3qELf27nLof99ddfNXToUIWEhCggIECDBg1Sdna2Tp06pd69eyswMFCBgYF6+umnC/xM8vPzNWXKFNWvX18+Pj4KDQ3VoEGDdPLkyRLHGRYWJkkFRl2ysrI0duxY1apVS97e3oqIiNDTTz9tO3bn7rNauXKltmzZYjt2q1atklT88+zcfWHz5s1T/fr15e3trWXLlkmSDh48qIcfflihoaHy9vZW/fr1NWvWrEvu0+7duyVJt9xyS4Fl7u7uCgoKsr3v27evqlevXqDdxe4/mjdvnurUqSMfHx9FR0fr559/LnTdkp7Phd3DlpycrH79+qlKlSry9vZWeHi4OnbsqL1799rabNy4UXFxcQoODpavr69q1Kihhx9+2K7vwu5h+/3333XPPffI399f5cqVU+vWrQtcJnvu/F29erWGDx+ukJAQlS1bVvfff7+OHTtW6HE636+//qqUlBS1adOmwLK3335b9evXV5kyZRQYGKgbb7xR8+fPt2tT3HMhMzNT48aNU+3ateXj46Pw8HA98MADtnNCuvxzc8mSJWrQoIFtu+fOzwv376abbpKPj49q1qyp9957r9DjsHz5ct16660KCAhQuXLlVKdOnQJfKnh6eqpVq1b68ssviz6gAC4bI2wALktqaqpSUlIK1Ofk5Fxy3XHjxik+Pl6PPPKImjdvrrS0NG3cuFGbNm3SXXfdpUGDBunQoUNavnx5gUv4DMPQfffdp5UrV6p///5q0qSJvvvuO40cOVIHDx60u6ytb9++Wrhwof71r3+pRYsW+umnn9S+ffsi4+rSpYsiIyP1yiuv2P7wWb58uf755x/169dPYWFh2rJli2bOnKktW7Zo3bp1Bf4g7tatm6KiojRx4kR98803eumll1ShQgW99957uvPOO/Xqq69q3rx5euqpp3TTTTfp9ttvv+ixeuSRRzR37lw9+OCDGjFihNavX6/4+HglJibqiy++kCR9/PHHmjlzpjZs2KAPPvhAknTzzTdf8udQUo8//rjCwsI0fvx4rVu3TjNnzlRAQIDWrFmjqlWr6pVXXtG3336r119/XQ0aNFDv3r1t6w4aNEhz5sxRv379NHToUO3Zs0fTpk3T77//rtWrVxfrEroTJ05IsiZ/Bw8e1IQJE+Tj46OuXbva2uTn5+u+++7Tr7/+qoEDByoqKkp//fWX3nzzTe3YsUNLlixRSEiIPv74Y7388stKT09XfHy8JOulo5dznknWyyoXLlyoIUOGKDg4WNWrV9eRI0fUokUL2x/NISEhWrp0qfr376+0tDQ98cQTRe5jtWrVJFkTq1tuuaVULwH86aef9Omnn2ro0KHy9vbWO++8o7Zt22rDhg0F7j8tzfO5c+fO2rJlix5//HFVr15dR48e1fLly5WUlGR7f/fddyskJETPPvusAgICtHfvXi1evPii+7Nlyxbddttt8vf319NPPy1PT0+99957atWqlX766SfFxMTYtX/88ccVGBiosWPHau/evZoyZYqGDBmiTz/99KLbWbNmjSwWi5o2bWpX//7772vo0KF68MEHNWzYMGVmZurPP//U+vXr9dBDD0lSsc+FvLw8dejQQStWrFD37t01bNgwnT59WsuXL9fff/+tmjVrXva5+euvv2rx4sV67LHH5Ofnp7feekudO3dWUlKSLfH/66+/bMd+3Lhxys3N1dixYxUaGlrgWHfo0EGNGjXSiy++KG9vb+3atUurV68ucLyio6P15ZdfKi0tTf7+/hc9tgCKyQCAYpg9e7Yh6aKv+vXr261TrVo1o0+fPrb3jRs3Ntq3b3/R7QwePNgo7KNpyZIlhiTjpZdesqt/8MEHDYvFYuzatcswDMNISEgwJBlPPPGEXbu+ffsakoyxY8fa6saOHWtIMnr06FFge2fOnClQ95///MeQZPz8888F+hg4cKCtLjc316hSpYphsViMiRMn2upPnjxp+Pr62h2TwmzevNmQZDzyyCN29U899ZQhyfjxxx9tdX369DHKli170f4KU79+faNly5aFLrvw53buZx8XF2fk5+fb6mNjYw2LxWI8+uijtrpz+35+37/88oshyZg3b57ddpYtW1Zo/YXOHeMLXwEBAcayZcvs2n788ceGm5ub8csvv9jVz5gxw5BkrF692lbXsmXLAudscc8zwzAMSYabm5uxZcsWu7b9+/c3wsPDjZSUFLv67t27G+XLly/03DonPz/faNmypSHJCA0NNXr06GFMnz7d2LdvX4G2ffr0MapVq1ag/tzxOt+5Y7Zx40Zb3b59+wwfHx/j/vvvL7BuSc/nPXv2GJKM2bNn29pIMl5//fUi9/mLL74wJBm//fZbkW3O7cP5v7+dOnUyvLy8jN27d9vqDh06ZPj5+Rm33367re7c+dumTRu78/fJJ5803N3djVOnTl10u7169TKCgoIK1Hfs2LHA+XOh4p4Ls2bNMiQZkydPLtDHuZgv99z08vKyq/vjjz8MScbbb79tq+vUqZPh4+Njd35t3brVcHd3tzuH3nzzTUOScezYsYvur2EYxvz58w1Jxvr16y/ZFkDxcEkkgMsyffp0LV++vMCrUaNGl1w3ICBAW7Zs0c6dOy97u99++63c3d01dOhQu/oRI0bIMAwtXbpUkmyX/Dz22GN27R5//PEi+y7s3hRfX1/bvzMzM5WSkqIWLVpIkjZt2lSg/SOPPGL7t7u7u2688UYZhqH+/fvb6gMCAlSnTh39888/RcYiWfdVkoYPH25XP2LECEnSN998c9H1HaV///52I4sxMTEF9vHcvp+/j4sWLVL58uV11113KSUlxfaKjo5WuXLltHLlymJt//PPP9fy5cv1/fffa/bs2apdu7Y6d+6sNWvW2G0rKipKdevWtdvWnXfeKUmX3FZxz7NzWrZsqXr16tneG4ahzz//XPfee68Mw7CLIS4uTqmpqYWeP+dYLBZ99913eumllxQYGKj//Oc/Gjx4sKpVq6Zu3bpd0T1ssbGxio6Otr2vWrWqOnbsqO+++055eXl2bUvrfD53f+WqVauKvPz13H2EX3/9dbFG6iXriNT333+vTp066YYbbrDVh4eH66GHHtKvv/6qtLQ0u3UGDhxod/7edtttysvL0759+y66rePHjyswMLDQuA8cOKDffvut0PUu51z4/PPPFRwcXOjn1LmYL/fcbNOmjd3EH40aNZK/v7/t55WXl6fvvvtOnTp1UtWqVW3toqKiFBcXV2BfJenLL79Ufn5+oft7zrljVdiVGABKhoQNwGVp3ry52rRpU+BV2B80F3rxxRd16tQp1a5dWw0bNtTIkSP1559/Fmu7+/btU6VKleTn52dXHxUVZVt+rnRzc1ONGjXs2tWqVavIvi9sK1kvvxs2bJhCQ0Pl6+urkJAQW7vU1NQC7c//g0eSypcvLx8fH7tJMs7VX+q+rXP7cGHMYWFhCggIuOQfmI5S2D5KUkRERIH68/dx586dSk1NVcWKFRUSEmL3Sk9P19GjR4u1/dtvv11t2rTRXXfdpb59+2rFihXy8/Oz+yN3586d2rJlS4Ht1K5dW5Iuua3inmfnXHjuHDt2TKdOndLMmTMLxNCvX79ixeDt7a3nnntOiYmJOnTokP7zn/+oRYsWtksvSyoyMrJAXe3atXXmzJkC93KV1vns7e2tV199VUuXLlVoaKhuv/12vfbaa0pOTra1admypTp37qzx48crODhYHTt21OzZs+3u17zQsWPHdObMGdWpU6fAsqioKNtMohfbp3OfWcW5j9K44B4xSXrmmWdUrlw5NW/eXJGRkRo8eLDdJYKXcy7s3r1bderUueglsJd7bl64v+f2+dz+Hjt2TGfPni30vLjwuHbr1k233HKLHnnkEYWGhqp79+5auHBhocnbuWPlrI8JAVwR97ABuGpuv/127d69W19++aW+//57ffDBB3rzzTc1Y8YMu2/0r7bzR9PO6dq1q9asWaORI0eqSZMmtunk27ZtW+gfKe7u7sWqkwr/468wzvYHT1H7U1j9+fuYn5+vihUrat68eYWuHxISUqJ4ypUrp5iYGH355Ze22T3z8/PVsGFDTZ48udB1Lkwur9SF5865c6NXr17q06dPoesUZzT6nPDwcHXv3l2dO3dW/fr1tXDhQs2ZM0ceHh5Fnh8XjpaVRGmez0888YTuvfdeLVmyRN99951eeOEFxcfH68cff1TTpk1lsVj02Wefad26dfrvf/+r7777Tg8//LAmTZqkdevWqVy5cle8P1cSf1BQUKFJXVRUlLZv366vv/5ay5Yt0+eff6533nlHY8aM0fjx40v9XLhcV/r5cz5fX1/9/PPPWrlypb755hstW7ZMn376qe688059//33dts6d6wuTO4BlBwJG4CrqkKFCurXr5/69eun9PR03X777Ro3bpwtYSvqj9Bq1arphx9+0OnTp+2+YT73IOFzkzVUq1ZN+fn52rNnj903x7t27Sp2jCdPntSKFSs0fvx4u4fAluRSzpI4tw87d+60fXsuWScwOHXqlG1fXUXNmjX1ww8/6JZbbik0Ob4Subm5kqT09HSVLVtWNWvW1B9//KHWrVuXKOEt7nlWlJCQEPn5+SkvL6/QWQVLytPTU40aNdLOnTuVkpKisLAwBQYGFnqJZFEjsIWdvzt27FCZMmVKnDQXV82aNTVixAiNGDFCO3fuVJMmTTRp0iR98skntjYtWrRQixYt9PLLL2v+/Pnq2bOnFixYUOiXOSEhISpTpoy2b99eYNm2bdvk5uZWasl53bp1NW/ePKWmptpGlc8pW7asunXrpm7duik7O1sPPPCAXn75ZY0aNeqyzoWaNWtq/fr1ysnJKXICnis9Ny8UEhIiX1/fQs+Lwo6rm5ubWrdurdatW2vy5Ml65ZVX9Nxzz2nlypV2+7dnzx65ubnZRrUBXDkuiQRw1Vw4JX65cuVUq1Ytu0ufzj0D7cI/RNu1a6e8vDxNmzbNrv7NN9+UxWLRPffcI0m2ey/eeecdu3Zvv/12seM8923xhd9ET5kypdh9XIl27doVur1zo0YXm/HSGXXt2lV5eXmaMGFCgWW5ubklvi/rxIkTWrNmjcLCwlSxYkXbtg4ePKj333+/QPuzZ88qIyPjon0W9zwriru7uzp37qzPP/9cf//9d4Hll5pGfufOnUpKSipQf+rUKa1du1aBgYG25KpmzZpKTU21u6z48OHDtllEL7R27Vq7++f279+vL7/8UnfffXeRozFX6syZM8rMzLSrq1mzpvz8/Gy/9ydPnizwu3buuXpFXRbp7u6uu+++W19++aXd4wGOHDmi+fPn69Zbby21GQpjY2NlGIYSEhLs6i/8PPPy8lK9evVkGIZycnIu61zo3LmzUlJSCpx30v8+h6703LyQu7u74uLitGTJErtzLjExUd99951d23MztJ6vqJ9RQkKC6tevXyC5BVByjLABuGrq1aunVq1aKTo6WhUqVNDGjRv12Wef2d2Xc25ShKFDhyouLk7u7u7q3r277r33Xt1xxx167rnntHfvXjVu3Fjff/+9vvzySz3xxBO2m+ujo6PVuXNnTZkyRcePH7dN679jxw5JxbvM0N/f33avTU5OjipXrqzvv/9ee/bsccBRKahx48bq06ePZs6cqVOnTqlly5basGGD5s6dq06dOumOO+64KnGUlpYtW2rQoEGKj4/X5s2bdffdd8vT01M7d+7UokWLNHXqVD344IOX7Oezzz5TuXLlZBiGDh06pA8//FAnT57UjBkzbD/Xf/3rX1q4cKEeffRRrVy5Urfccovy8vK0bds2LVy4UN99951uvPHGIrdR3PPsYiZOnKiVK1cqJiZGAwYMUL169XTixAlt2rRJP/zwQ6F//J7zxx9/6KGHHtI999yj2267TRUqVNDBgwc1d+5cHTp0SFOmTLElV927d9czzzyj+++/X0OHDtWZM2f07rvvqnbt2oVObNKgQQPFxcXZTesvSePHj7/kPpXUjh071Lp1a3Xt2lX16tWTh4eHvvjiCx05ckTdu3eXJM2dO1fvvPOO7r//ftWsWVOnT5/W+++/L39/f9uXF4V56aWXbM8Ge+yxx+Th4aH33ntPWVlZeu2110ptH2699VYFBQXphx9+sE1eI0l33323wsLCdMsttyg0NFSJiYmaNm2a2rdvbxsBK+650Lt3b3300UcaPny4NmzYoNtuu00ZGRn64Ycf9Nhjj6ljx46lcm5eaPz48Vq2bJluu+02PfbYY8rNzbU9W+78LwJefPFF/fzzz2rfvr2qVaumo0eP6p133lGVKlVsz66UrI93+emnnwpM+gTgCl3FGSkBuLBzU2MXNfV2YVOkXzg9/EsvvWQ0b97cCAgIMHx9fY26desaL7/8spGdnW1rk5ubazz++ONGSEiIYbFY7KaWPn36tPHkk08alSpVMjw9PY3IyEjj9ddft5uq2zAMIyMjwxg8eLBRoUIFo1y5ckanTp2M7du3G5LspiU/N4V5YVNVHzhwwLj//vuNgIAAo3z58kaXLl2MQ4cOFflogAv7KGq6/cKOU2FycnKM8ePHGzVq1DA8PT2NiIgIY9SoUUZmZmaxtnMpJZnW/8Kf/eXu+8yZM43o6GjD19fX8PPzMxo2bGg8/fTTxqFDhy4aa2HT+pctW9aIjY01Fi5cWKB9dna28eqrrxr169c3vL29jcDAQCM6OtoYP368kZqaamtX1M+iuOeZJGPw4MGFxnzkyBFj8ODBRkREhOHp6WmEhYUZrVu3NmbOnHnRfT1y5IgxceJEo2XLlkZ4eLjh4eFhBAYGGnfeeafx2WefFWj//fffGw0aNDC8vLyMOnXqGJ988kmR0/oPHjzY+OSTT4zIyEjD29vbaNq0qbFy5Uq7dld6Pl84rX9KSooxePBgo27dukbZsmWN8uXLGzExMXY/t02bNhk9evQwqlatanh7exsVK1Y0OnToYPcIgnP7cP7v3rl14+LijHLlyhllypQx7rjjDmPNmjV2bYo6f1euXGlIKnAMCjN06FCjVq1adnXvvfeecfvttxtBQUGGt7e3UbNmTWPkyJF255hhFP9cOHPmjPHcc8/ZfufDwsKMBx980O6xBVd6bl74u20YhvHTTz8Z0dHRhpeXl3HDDTcYM2bMKHAOrVixwujYsaNRqVIlw8vLy6hUqZLRo0cPY8eOHXZ9LV261JBk7Ny585LHFEDxWQyjBHefAoCL2bx5s5o2bapPPvlEPXv2NDscAC7kn3/+Ud26dbV06VK1bt3a7HCcVqdOnWSxWIq8LBdAyZCwAbjmnD17tsDkFn379tXHH3+svXv3lvpMgQCuff/+97+1a9cuLV++3OxQnFJiYqIaNmyozZs3q0GDBmaHA1xTSNgAXHPGjx+vhIQE3XHHHfLw8NDSpUu1dOlSDRw4UO+9957Z4QEAABQbCRuAa87y5cs1fvx4bd26Venp6apatar+9a9/6bnnnrvog2kBAACcDQkbAAAAADgpnsMGAAAAAE6KhA0AAAAAnBQ3c5RQfn6+Dh06JD8/v2I9iBcAAADAtckwDJ0+fVqVKlWSm1vpjomRsJXQoUOHmBocAAAAgM3+/ftVpUqVUu2ThK2E/Pz8JFl/KP7+/iZH45w2J29Wy9kt9VO/n9QkrInZ4QCA09u8WWrZUvrpJ6lJE7OjAQAUV1pamiIiImw5QmkiYSuhc5dB+vv7k7AVoVxGOclHKudXjmMEAMVQrtz/Sj42AcD1OOJWKSYdAQAAAAAnRcIGAAAAAE7KKRK26dOnq3r16vLx8VFMTIw2bNhw0faLFi1S3bp15ePjo4YNG+rbb7+1Wz5u3DjVrVtXZcuWVWBgoNq0aaP169fbtTlx4oR69uwpf39/BQQEqH///kpPTy/1fQMAAACAkrIYhmGYGcCnn36q3r17a8aMGYqJidGUKVO0aNEibd++XRUrVizQfs2aNbr99tsVHx+vDh06aP78+Xr11Ve1adMmNWjQQJI0f/58VaxYUTfccIPOnj2rN998U4sWLdKuXbsUEhIiSbrnnnt0+PBhvffee8rJyVG/fv100003af78+cWKOy0tTeXLl1dqair3ZxUhOy9bRzOOqmLZivJy9zI7HABwetnZ0tGjUsWKkhcfm8BVk5eXp5ycHLPDgBNzd3eXh4dHkfeoOTI3MD1hi4mJ0U033aRp06ZJsj7fLCIiQo8//rieffbZAu27deumjIwMff3117a6Fi1aqEmTJpoxY0ah2zh3AH/44Qe1bt1aiYmJqlevnn777TfdeOONkqRly5apXbt2OnDggCpVqnTJuEnYAAAAXF96eroOHDggk/8khgsoU6aMwsPD5VXIN2qOzA1MnSUyOztbCQkJGjVqlK3Ozc1Nbdq00dq1awtdZ+3atRo+fLhdXVxcnJYsWVLkNmbOnKny5curcePGtj4CAgJsyZoktWnTRm5ublq/fr3uv//+Av1kZWUpKyvL9j4tLa3Y+3m9+ufkP3rmh2f0aptXdUPgDWaHAwBO759/pGeekV59VbqBj03A4fLy8nTgwAGVKVNGISEhDpnhD67PMAxlZ2fr2LFj2rNnjyIjI0v94dgXY2rClpKSory8PIWGhtrVh4aGatu2bYWuk5ycXGj75ORku7qvv/5a3bt315kzZxQeHq7ly5crODjY1seFl1t6eHioQoUKBfo5Jz4+XuPHj7+s/bvenco8pc+2fqZRt466dGMAgE6dkj77TBrFxyZwVeTk5MgwDIWEhMjX19fscODEfH195enpqX379ik7O1s+Pj5XbdtOMemII9xxxx3avHmz1qxZo7Zt26pr1646evRoifsbNWqUUlNTba/9+/eXYrQAAAAwCyNrKI6rOapmt11Ttvr/goOD5e7uriNHjtjVHzlyRGFhYYWuExYWVqz2ZcuWVa1atdSiRQt9+OGH8vDw0Icffmjr48LkLTc3VydOnChyu97e3raHZPOwbAAAAABXg6kJm5eXl6Kjo7VixQpbXX5+vlasWKHY2NhC14mNjbVrL0nLly8vsv35/Z67By02NlanTp1SQkKCbfmPP/6o/Px8xcTElHR3AAAAAKBUmX5J5PDhw/X+++9r7ty5SkxM1L///W9lZGSoX79+kqTevXvbTUoybNgwLVu2TJMmTdK2bds0btw4bdy4UUOGDJEkZWRkaPTo0Vq3bp327dunhIQEPfzwwzp48KC6dOkiSYqKilLbtm01YMAAbdiwQatXr9aQIUPUvXv3Ys0QieKp5FdJr9z5iir5cUwBoDgqVZJeecVaAsC1plWrVnriiSfMDqNI1atX15QpU8wOowBTJx2RrNP0Hzt2TGPGjFFycrKaNGmiZcuW2SYWSUpKsrte9Oabb9b8+fP1/PPPa/To0YqMjNSSJUtsz2Bzd3fXtm3bNHfuXKWkpCgoKEg33XSTfvnlF9WvX9/Wz7x58zRkyBC1bt1abm5u6ty5s956662ru/PXuLByYRp1G3fOA0BxhYUx4QiAS+vbt69OnTpV5CzpuLaYnrBJ0pAhQ2wjZBdatWpVgbouXbrYRssu5OPjo8WLF19ymxUqVCj2Q7JRMqcyT+nnfT/r9mq3K8AnwOxwAMDpnTol/fyzdPvtUkCA2dEAQMlkZ2cX+qwylIzpl0Ti2vXPyX/UcUFH/XPyH7NDAQCX8M8/UseO1hIASuqnn35S8+bN5e3trfDwcD377LPKzc2VZH30VUBAgPLy8iRJmzdvlsVi0bPPPmtb/5FHHlGvXr1s73/99Vfddttt8vX1VUREhIYOHaqMjAzb8urVq2vChAnq3bu3/P39NXDgwCJjy83N1ZAhQ1S+fHkFBwfrhRdesHto+cmTJ9W7d28FBgaqTJkyuueee7Rz507b8nHjxqlJkyZ2fU6ZMkXVq1e3ve/bt686deqkN954Q+Hh4QoKCtLgwYOVk5Nja3P06FHde++98vX1VY0aNTRv3rxiHt2rzylG2AAAAACncfiw9XW+wECpRg0pM1PaurXgOs2aWcvt26XzkhlJUvXqUoUK0rFj0oWPhvLzkyIjSy30gwcPql27durbt68++ugjbdu2TQMGDJCPj4/GjRun2267TadPn9bvv/+uG2+8UT/99JOCg4Ptrmr76aef9Mwzz0iSdu/erbZt2+qll17SrFmzdOzYMdvVcbNnz7at88Ybb2jMmDEaO3bsReObO3eu+vfvrw0bNmjjxo0aOHCgqlatqgEDBkiyJls7d+7UV199JX9/fz3zzDNq166dtm7dKk9Pz2Ifh5UrVyo8PFwrV67Url271K1bNzVp0sRuO4cOHdLKlSvl6empoUOHXtEjwBzKQImkpqYakozU1FSzQ3FaCYcSDI2TkXAowexQAMAlJCQYhmQtATje2bNnja1btxpnz561XzB2rPWX8fxXz57WZTt3Flx2/p/ULVoUXPbxx9Zl06YVXHb33Zcdd58+fYyOHTsWumz06NFGnTp1jPz8fFvd9OnTjXLlyhl5eXmGYRhGs2bNjNdff90wDMPo1KmT8fLLLxteXl7G6dOnjQMHDhiSjB07dhiGYRj9+/c3Bg4caLeNX375xXBzc7Mdt2rVqhmdOnW6ZNwtW7Y0oqKi7GJ75plnjKioKMMwDGPHjh2GJGP16tW25SkpKYavr6+xcOFCwzAMY+zYsUbjxo3t+n3zzTeNatWq2R2fatWqGbm5uba6Ll26GN26dTMMwzC2b99uSDI2bNhgW56YmGhIMt58880i4y/yfDEcmxswwgYAAACcb9Ag6b777OsCA61llSrSeY+GKmDOnMJH2CSpa1fpwkdR+fldSaQFJCYmKjY21u5h4LfccovS09N14MABVa1aVS1bttSqVas0YsQI/fLLL4qPj9fChQv166+/6sSJE6pUqZIi/3/U748//tCff/5pd8mgYRjKz8/Xnj17FBUVJUm68cYbixVfixYt7GKLjY3VpEmTlJeXp8TERHl4eNg9ZisoKEh16tRRYmLiZR2H+vXry93d3fY+PDxcf/31l+0YeXh4KDo62ra8bt26CnDSm4dJ2OAwPh4+qhdSTz4ePmaHAgAuwcdHqlfPWgIwUXi49VUYH5//Xf5YmDp1il4WEmJ9maxVq1aaNWuW/vjjD3l6eqpu3bpq1aqVVq1apZMnT6ply5a2tunp6Ro0aJCGDh1aoJ+qVava/l22bNmrErubm5vdPW+S7O5NO+fCyyctFovy8/MdGpujkLBdx5KSkpSSklIqfQUHB9v90kpSvZB62vLYllLpHwCuB/XqSVv42ARwBaKiovT555/LMAzbSNbq1avl5+enKlWqSJLtPrY333zTlpy1atVKEydO1MmTJzVixAhbf82aNdPWrVtVq1atUolv/fr1du/XrVunyMhIubu7KyoqSrm5uVq/fr1uvvlmSdLx48e1fft21atXT5IUEhKi5ORku/3bvHnzZcVQt25d5ebmKiEhQTfddJMkafv27Tp16tSV7ZyDkLBdp5KSklSnbpQyz54plf58fMto+7bEAkkbAAAASl9qamqBRCUoKEiPPfaYpkyZoscff1xDhgzR9u3bNXbsWA0fPtz2bOPAwEA1atRI8+bN07Rp0yRJt99+u7p27aqcnBy7EbZnnnlGLVq00JAhQ/TII4+obNmy2rp1q5YvX25b93IkJSVp+PDhGjRokDZt2qS3335bkyZNkiRFRkaqY8eOGjBggN577z35+fnp2WefVeXKldWxY0dJ1sTy2LFjeu211/Tggw9q2bJlWrp0qfz9/YsdQ506ddS2bVsNGjRI7777rjw8PPTEE0/I19f3svfnaiBhu06lpKQo8+wZBXUYIc+giCvqK+f4fh3/epJSUlLsErbNyZt1++zb9XO/n9UkrMkVRgwA177Nm63PYPv5Z+mCWasBwM6qVavUtGlTu7r+/fvrgw8+0LfffquRI0eqcePGqlChgvr376/nn3/erm3Lli21efNmtWrVSpL1GcX16tXTkSNHVOe8yzobNWqkn376Sc8995xuu+02GYahmjVrqlu3biWKu3fv3jp79qyaN28ud3d3DRs2zO4xALNnz9awYcPUoUMHZWdn6/bbb9e3335ru8QxKipK77zzjl555RVNmDBBnTt31lNPPaWZM2deVhyzZ8/WI488opYtWyo0NFQvvfSSXnjhhRLtk6NZjAsvAkWxpKWlqXz58kpNTb2sjN5ZbNq0SdHR0QrrM0XeYVc2xJ2VvEvJc59QQkKCmp13Tfemw5sUPTNaCQMT1Cz8Itd6AwAkSZs2SdHR1vkMLnaLDIDSkZmZqT179qhGjRry4eZRXMLFzhdH5gY8OBsAAAAAnBQJGwAAAAA4KRI2AAAAAHBSJGxwmLrBdZUwMEF1g+uaHQoAuIS6da33r9XlYxMA8P+YJRIOU8azDJONAMBlKFOGyUYAAPYYYYPDJKUmafA3g5WUmmR2KADgEpKSpMGDrSUAABIJGxwo5UyK3tn4jlLOpJgdCgC4hJQU6Z13rCUAABIJGwAAAAA4LRI2AAAAAHBSJGwAAAAAHG7VqlWyWCw6deqU2aEUas6cOQoICDA7jAJI2OAwFctW1JMtnlTFshXNDgUAXELFitKTT1pLAChKq1at9MQTTxSod9aEA1eGaf3hMFX8q2hy3GSzwwAAl1GlijSZj00ALi4vL08Wi0VubowNlQaOIhwmPTtda/evVXp2utmhAIBLSE+X1q61lgBwpfr27atOnTpp/PjxCgkJkb+/vx599FFlZ2fb2rRq1UpDhgzRkCFDVL58eQUHB+uFF16QYRi2NllZWXrqqadUuXJllS1bVjExMVq1apVt+bmRva+++kr16tWTt7e3ki7yfJLVq1erUaNG8vHxUYsWLfT333/bLf/8889Vv359eXt7q3r16po0aZLdcovFoiVLltjVBQQEaM6cOZKkvXv3ymKxaPHixbrjjjtUpkwZNW7cWGvXrrVbZ86cOapatarKlCmj+++/X8ePHy/OYb3qGGGDw+w4vkM3z7pZCQMTeIA2ABTDjh3SzTdLCQk8QBsw1eHD1tf5AgOlGjWkzExp69aC65z7pd2+XcrIsF9WvbpUoYJ07Ji0f7/9Mj8/KTKy1EK/0IoVK+Tj46NVq1Zp79696tevn4KCgvTyyy/b2sydO1f9+/fXhg0btHHjRg0cOFBVq1bVgAEDJElDhgzR1q1btWDBAlWqVElffPGF2rZtq7/++kuR/x/7mTNn9Oqrr+qDDz5QUFCQKl7k2u6RI0dq6tSpCgsL0+jRo3Xvvfdqx44d8vT0VEJCgrp27apx48apW7duWrNmjR577DEFBQWpb9++l7Xvzz33nN544w1FRkbqueeeU48ePbRr1y55eHho/fr16t+/v+Lj49WpUyctW7ZMY8eOvfwDfBWQsAEAAADne+89afx4+7qePaVPPpEOHJCiowuuc25Eqm9fad06+2Uffyz16iUtXCgNGWK/7O67pe++K7XQL+Tl5aVZs2apTJkyql+/vl588UWNHDlSEyZMsF2yGBERoTfffFMWi0V16tTRX3/9pTfffFMDBgxQUlKSZs+eraSkJFWqVEmS9NRTT2nZsmWaPXu2XnnlFUlSTk6O3nnnHTVu3PiSMY0dO1Z33XWXJGuyWKVKFX3xxRfq2rWrJk+erNatW+uFF16QJNWuXVtbt27V66+/ftkJ21NPPaX27dtLksaPH6/69etr165dqlu3rqZOnaq2bdvq6aeftm1nzZo1WrZs2WVt42ogYQMAAADON2iQdN999nWBgdayShXrMHhR5swpfIRNkrp2lWJj7Zf5+V1JpJfUuHFjlSlTxvY+NjZW6enp2r9/v6pVqyZJatGihSwWi12bSZMmKS8vT3/99Zfy8vJUu3Ztu36zsrIUFBRke+/l5aVGjRoVK6bY845BhQoVVKdOHSUmJkqSEhMT1bFjR7v2t9xyi6ZMmaK8vDy5u7sXc89lF094eLgk6ejRo6pbt64SExN1//33F4iLhA0AAABwduHh1ldhfHwufs1ynTpFLwsJsb6ukL+/v1JTUwvUnzp1SuXLl7/i/s+Xnp4ud3d3JSQkFEiWypUrZ/u3r6+vXdLnSBaLxe4eO8k6wnchT09Pu3UkKT8/37HBOQAJGxzGw81DwWWC5eHGaQYAxeHhIQUHW0sAKEqdOnX0/fffF6jftGlTgZGwP/74Q2fPnpWvr68kad26dSpXrpwiIiJsbdavX2+3zrp16xQZGSl3d3c1bdpUeXl5Onr0qG677bZSiX/dunWqWrWqJOnkyZPasWOHoqKiJElRUVFavXq1XfvVq1erdu3atoQxJCREh8+7x3Dnzp06c+bMZcUQFRVV6H47I/5LgMM0Cm2kYyOPmR0GALiMRo2scxIAwMX8+9//1rRp0zR06FA98sgj8vb21jfffKP//Oc/+u9//2vXNjs7W/3799fzzz+vvXv3auzYsRoyZIjdlPtJSUkaPny4Bg0apE2bNuntt9+2zcxYu3Zt9ezZU71799akSZPUtGlTHTt2TCtWrFCjRo1s94hdjhdffFFBQUEKDQ3Vc889p+DgYHXq1EmSNGLECN10002aMGGCunXrprVr12ratGl65513bOvfeeedmjZtmmJjY5WXl6dnnnnGbjStOIYOHapbbrlFb7zxhjp27KjvvvvOKS+HlJjWHwAAAHApN9xwg37++Wdt27ZNbdq0UUxMjBYuXKhFixapbdu2dm1bt26tyMhI3X777erWrZvuu+8+jRs3zq5N7969dfbsWTVv3lyDBw/WsGHDNHDgQNvy2bNnq3fv3hoxYoTq1KmjTp066bfffrONkl2uiRMnatiwYYqOjlZycrL++9//ysvLS5LUrFkzLVy4UAsWLFCDBg00ZswYvfjii3YTjkyaNEkRERG67bbb9NBDD+mpp56yu0+vOFq0aKH3339fU6dOVePGjfX999/r+eefL9H+OJrFuPACUBRLWlqaypcvr9TUVPn7+5sdzmXbtGmToqOjFdZnirzDal1RX1nJu5Q89wklJCSo2XnXdG85ukUdF3TUl92/VP2K9a80ZAC45m3ZInXsKH35pVSfj03A4TIzM7Vnzx7VqFFDPj4+ZodT6vr27atTp04VeGbZ+Vq1aqUmTZpoypQpVy0uV3Wx88WRuQEjbHCYrLws7T65W1l5WWaHAgAuIStL2r3bWgIAIJGwAQAAAIDTYtIRAAAA4Bo0Z86cS7ZZtWqVw+PAlWGEDQAAAACcFAkbHKZWhVpa1nOZalW4sklNAOB6UauWtGyZtQQAQOKSSDiQv7e/4mrFmR0GALgMf38pjo9NAMB5GGGDwxw+fVjjVo3T4dOHL90YAKDDh6Vx46wlAAASCRsc6HD6YY3/abwOp/OXBwAUx+HD0vjxJGwAgP8hYQMAAAAAJ0XCBgAAAABOioQNAAAAcCF9+/aVxWLRo48+WmDZ4MGDZbFY1Ldv36sfGByChA0OE+gTqJ4NeyrQJ9DsUADAJQQGSj17WksAuJiIiAgtWLBAZ8+etdVlZmZq/vz5qlq1qomRobSRsMFhagTW0CcPfKIagTXMDgUAXEKNGtInn1hLALiYZs2aKSIiQosXL7bVLV68WFWrVlXTpk1tdfn5+YqPj1eNGjXk6+urxo0b67PPPrMtz8vLU//+/W3L69Spo6lTp9ptq2/fvurUqZPeeOMNhYeHKygoSIMHD1ZOTo7jdxQ8hw2Ok5mbqQNpB1TFv4p8PHzMDgcAnF5mpnTggFSliuTDxyZgmsOHC87WGhho/TIlM1PaurXgOs2aWcvt26WMDPtl1atLFSpIx45J+/fbL/PzkyIjSxbnww8/rNmzZ6tnz56SpFmzZqlfv35atWqVrU18fLw++eQTzZgxQ5GRkfr555/Vq1cvhYSEqGXLlsrPz1eVKlW0aNEiBQUFac2aNRo4cKDCw8PVtWtXWz8rV65UeHi4Vq5cqV27dqlbt25q0qSJBgwYULLgUWwkbHCYrce2KnpmtBIGJqhZeDOzwwEAp7d1qxQdLSUk/O+PPwBX33vvWR+xcb6ePa0j4AcOWH9PL2QY1rJvX2ndOvtlH38s9eolLVwoDRliv+zuu6XvvitZnL169dKoUaO0b98+SdLq1au1YMECW8KWlZWlV155RT/88INiY2MlSTfccIN+/fVXvffee2rZsqU8PT01/rydrVGjhtauXauFCxfaJWyBgYGaNm2a3N3dVbduXbVv314rVqwgYbsKSNgAAACA8wwaJN13n33duXtLq1SxfqlSlDlzCh9hk6SuXaX/z5ts/PxKHmdISIjat2+vOXPmyDAMtW/fXsHBwbblu3bt0pkzZ3TXXXfZrZednW132eT06dM1a9YsJSUl6ezZs8rOzlaTJk3s1qlfv77c3d1t78PDw/XXX3+VPHgUGwkbAAAAcJ7wcOurMD4+Fx8Br1On6GUhIdZXaXr44Yc15P+H7aZPn263LD09XZL0zTffqHLlynbLvL29JUkLFizQU089pUmTJik2NlZ+fn56/fXXtX79erv2np6edu8tFovy8/NLdV9QOBI2AAAAwEW1bdtW2dnZslgsiouLs1tWr149eXt7KykpSS1btix0/dWrV+vmm2/WY489ZqvbvXu3Q2PG5SFhAwAAAFyUu7u7EhMTbf8+n5+fn5566ik9+eSTys/P16233qrU1FStXr1a/v7+6tOnjyIjI/XRRx/pu+++U40aNfTxxx/rt99+Uw2mq3UaJGxwmGbhzWSMNcwOAwBcRrNm/5u4AACKy9/fv8hlEyZMUEhIiOLj4/XPP/8oICBAzZo10+jRoyVJgwYN0u+//65u3brJYrGoR48eeuyxx7R06dKrFT4uwWIY/NdQEmlpaSpfvrxSU1Mv+kvirDZt2qTo6GiF9Zki77BaV9RXVvIuJc99QgkJCWrGtGYAAMBFZGZmas+ePapRo4Z8eJYGLuFi54sjcwMenA2H2Z6yXbEfxmp7ynazQwEAl7B9u3UGue18bAIA/h8JGxwmIydD6w6sU0ZOxqUbAwCUkWF9ftOFU4IDAK5fJGwAAAAA4KRI2AAAAADASZGwAQAAAICTImGDw1QPqK6P7/9Y1QOqmx0KALiE6tWljz+2lgAASDyHDQ5UwbeCejXqZXYYAOAyKlSQevGxCQA4DyNscJhjGcc0fcN0Hcs4ZnYoAOASjh2Tpk+3lgAASCRscKD9afs1ZOkQ7U/bb3YoAOAS9u+XhgyxlgAASCRsAAAAAM6zatUqWSwWnTp1SpI0Z84cBQQEmBrT9YyEDQAAAHAhffv2lcVi0aOPPlpg2eDBg2WxWNS3b99S2163bt20Y8eOUusPl4eEDQAAAHAxERERWrBggc6ePWury8zM1Pz581W1atVS3Zavr68qVqxYqn2i+EjY4DB+Xn66u+bd8vPyMzsUAHAJfn7S3XdbSwC4mGbNmikiIkKLFy+21S1evFhVq1ZV06ZNbXX5+fmKj49XjRo15Ovrq8aNG+uzzz6z6+vbb79V7dq15evrqzvuuEN79+61W37hJZG7d+9Wx44dFRoaqnLlyummm27SDz/8YLdO9erV9corr+jhhx+Wn5+fqlatqpkzZ5beAbiOMK0/HCYyKFLf9frO7DAAwGVERkrf8bEJmO7w6cM6nH7Yri7QJ1A1AmsoMzdTW49tLbBOs/BmkqTtKduVkZNht6x6QHVV8K2gYxnHCkzG5uflp8igyBLF+fDDD2v27Nnq2bOnJGnWrFnq16+fVq1aZWsTHx+vTz75RDNmzFBkZKR+/vln9erVSyEhIWrZsqX279+vBx54QIMHD9bAgQO1ceNGjRgx4qLbTU9PV7t27fTyyy/L29tbH330ke69915t377dbnRv0qRJmjBhgkaPHq3PPvtM//73v9WyZUvVqVOnRPt7vXKKEbbp06erevXq8vHxUUxMjDZs2HDR9osWLVLdunXl4+Ojhg0b6ttvv7Uty8nJ0TPPPKOGDRuqbNmyqlSpknr37q1Dhw7Z9VG9enVZLBa718SJEx2yf9ervPw8pWWlKS8/z+xQAMAl5OVJaWnWEoB53kt4T9Ezo+1eL6x8QZJ0IO1AgWXRM6Nt6/b9sm+BZd/utP6tunDLwgLLhiwdUuI4e/XqpV9//VX79u3Tvn37tHr1avU672GOWVlZeuWVVzRr1izFxcXphhtuUN++fdWrVy+99957kqR3331XNWvW1KRJk1SnTh317Nnzkve/NW7cWIMGDVKDBg0UGRmpCRMmqGbNmvrqq6/s2rVr106PPfaYatWqpWeeeUbBwcFauXJliff3emX6CNunn36q4cOHa8aMGYqJidGUKVMUFxen7du3F3qt7Jo1a9SjRw/Fx8erQ4cOmj9/vjp16qRNmzapQYMGOnPmjDZt2qQXXnhBjRs31smTJzVs2DDdd9992rhxo11fL774ogYMGGB778c1KKXqjyN/KHpmtBIGJti+dQIAFO2PP6ToaCkhQWrGxyZgmkHRg3Rfnfvs6gJ9AiVJVfyrKGFgQpHrzuk4p9ARNknqWr+rYiNi7ZZdya0jISEhat++vebMmSPDMNS+fXsFBwfblu/atUtnzpzRXXfdZbdedna27bLJxMRExcTE2C2PjbWP8ULp6ekaN26cvvnmGx0+fFi5ubk6e/askpKS7No1atTI9m+LxaKwsDAdPXq0RPt6PTM9YZs8ebIGDBigfv36SZJmzJihb775RrNmzdKzzz5boP3UqVPVtm1bjRw5UpI0YcIELV++XNOmTdOMGTNUvnx5LV++3G6dadOmqXnz5kpKSrIbpvXz81NYWJgD9w4AAACuJtwvXOF+4YUu8/HwuegX0XWCi77cL6RsiELKhlxxfOd7+OGHNWSIdZRu+vTpdsvS09MlSd98840qV65st8zb27vE23zqqae0fPlyvfHGG6pVq5Z8fX314IMPKjs7266dp6en3XuLxaL8/PwSb/d6ZeolkdnZ2UpISFCbNm1sdW5ubmrTpo3Wrl1b6Dpr1661ay9JcXFxRbaXpNTUVFkslgLPj5g4caKCgoLUtGlTvf7668rNzS2yj6ysLKWlpdm9AAAAADO1bdtW2dnZysnJUVxcnN2yevXqydvbW0lJSapVq5bdKyIiQpIUFRVV4HakdevWXXSbq1evVt++fXX//ferYcOGCgsLKzBRCUqPqSNsKSkpysvLU2hoqF19aGiotm3bVug6ycnJhbZPTk4utH1mZqaeeeYZ9ejRQ/7+/rb6oUOHqlmzZqpQoYLWrFmjUaNG6fDhw5o8eXKh/cTHx2v8+PGXs3sAAACAQ7m7uysxMdH27/P5+fnpqaee0pNPPqn8/HzdeuutSk1N1erVq+Xv768+ffro0Ucf1aRJkzRy5Eg98sgjSkhI0Jw5cy66zcjISC1evFj33nuvLBaLXnjhBUbOHMj0SyIdKScnR127dpVhGHr33Xftlg0fPtz270aNGsnLy0uDBg1SfHx8oUPEo0aNslsnLS3N9s0EAAAAYJbzByUuNGHCBIWEhCg+Pl7//POPAgIC1KxZM40ePVqSVLVqVX3++ed68skn9fbbb6t58+a26fiLMnnyZD388MO6+eabFRwcrGeeeYarzxzI1IQtODhY7u7uOnLkiF39kSNHiry3LCwsrFjtzyVr+/bt048//njRE1mSYmJilJubq7179xY61ai3t/cVXet7PWpYsaGOPnVUAT4BZocCAC6hYUPp6FHpgiv4AcDOpUbAlixZYvu3xWLRsGHDNGzYsCLbd+jQQR06dLCrOze/hCT17dvXbubI6tWr68cff7RrP3jwYLv3hV0iuXnz5ovGjcKZeg+bl5eXoqOjtWLFCltdfn6+VqxYUeTsNLGxsXbtJWn58uV27c8lazt37tQPP/ygoKCgS8ayefNmubm58RT3UuTp7qmQsiHydPe8dGMAgDw9pZAQawkAgOQEz2EbPny43n//fc2dO1eJiYn697//rYyMDFtW37t3b40aNcrWftiwYVq2bJkmTZqkbdu2ady4cdq4caNtdpycnBw9+OCD2rhxo+bNm6e8vDwlJycrOTnZNnPN2rVrNWXKFP3xxx/6559/NG/ePD355JPq1auXAgMDr/5BuEbtPrFb9/3nPu0+sdvsUADAJezeLd13n7UEAEBygnvYunXrpmPHjmnMmDFKTk5WkyZNtGzZMtvEIklJSXJz+19eefPNN2v+/Pl6/vnnNXr0aEVGRmrJkiVq0KCBJOngwYO2h/Y1adLEblsrV65Uq1at5O3trQULFmjcuHHKyspSjRo19OSTT9rdo4Yrl5qVqv/u+K/GtRpndigA4BJSU6X//lcaN87sSAAAzsL0hE2ShgwZYhshu9CqVasK1HXp0kVdunQptH316tVlGMZFt9esWbNLTlcKAAAAAGYz/ZJIAAAAwEyX+rIfkMw7T0jYAAAAcF0699yyc/McABdz5swZSZLnVZ4ZyikuicS1qbJfZU26e5Iq+1U2OxQAcAmVK0uTJllLAI7n4eGhMmXK6NixY/L09LSbNwE4xzAMnTlzRkePHlVAQECBB5Q7GgkbHCa0XKiGxzKRCwAUV2ioxPxXwNVjsVgUHh6uPXv2aN++fWaHAycXEBBQ5LOiHYmEDQ5z8uxJ/fDPD2pzQxsF+vK4BAC4lJMnpR9+kNq0kXjKDHB1eHl5KTIykssicVGenp5XfWTtHBI2OMyeU3vU9bOuShiYQMIGAMWwZ4/UtauUkEDCBlxNbm5u8vHxMTsMoFBcqAsAAAAAToqEDQAAAACcFAkbAAAAADgpEjY4jK+Hr5qGNZWvh6/ZoQCAS/D1lZo2tZYAAEhMOgIHigqJ0qZBm8wOAwBcRlSUtImPTQDAeRhhAwAAAAAnRcIGh/n98O/yfslbvx/+3exQAMAl/P675O1tLQEAkEjY4ECGDGXnZcuQYXYoAOASDEPKzraWAABIJGwAAAAA4LRI2AAAAADASZGwAQAAAICTYlp/OExUcJT+/vffuiHwBrNDAQCXEBUl/f23dAMfmwCA/0fCBofx9fRV/Yr1zQ4DAFyGr69Un49NAMB5uCQSDrPv1D498tUj2ndqn9mhAIBL2LdPeuQRawkAgETCBgc6fva4Pvz9Qx0/e9zsUADAJRw/Ln34obUEAEAiYQMAAAAAp0XCBgAAAABOioQNAAAAAJwUCRscJrRsqJ695VmFlg01OxQAcAmhodKzz1pLAAAkpvWHA1X2r6z4NvFmhwEALqNyZSmej00AwHkYYYPDnM46rVV7V+l01mmzQwEAl3D6tLRqlbUEAEAiYYMD7TyxU3fMvUM7T+w0OxQAcAk7d0p33GEtAQCQSNgAAAAAwGmRsAEAAACAkyJhAwAAAAAnRcIGh/F081Rlv8rydPM0OxQAcAmentaZIj352AQA/D+m9YfDNAxtqAPDD5gdBgC4jIYNpQN8bAIAzsMIGwAAAAA4KRI2OMxfR/5SlclV9NeRv8wOBQBcwl9/SVWqWEsAACQSNjhQTn6ODp4+qJz8HLNDAQCXkJMjHTxoLQEAkEjYAAAAAMBpkbABAAAAgJMiYQMAAAAAJ0XCBoeJrBCplX1WKrJCpNmhAIBLiIyUVq60lgAASDyHDQ7k5+2nVtVbmR0GALgMPz+pVSuzowAAOBNG2OAwB9MOatQPo3Qw7aDZoQCASzh4UBo1yloCACCRsMGBjmQc0cTVE3Uk44jZoQCASzhyRJo40VoCACCRsAEAAACA0yJhAwAAAAAnRcIGAAAAAE6KhA0OE+QbpP5N+yvIN8jsUADAJQQFSf37W0sAACSm9YcDVQuopg/u+8DsMADAZVSrJn3AxyYA4DyMsMFhzuac1ZajW3Q256zZoQCASzh7VtqyxVoCACCRsMGBElMS1eDdBkpMSTQ7FABwCYmJUoMG1hIAAImEDQAAAACcFgkbAAAAADgpEjYAAAAAcFIkbHAYiyzycveSRRazQwEAl2CxSF5e1hIAAIlp/eFATcObKuv5LLPDAACX0bSplMXHJgDgPIywAQAAAICTImGDwyQeS1Sz95op8RjzUwNAcSQmSs2aMa0/AOB/SNjgMGdzz+r35N91NpcnwAJAcZw9K/3+Ow/OBgD8DwkbAAAAADgpEjYAAAAAcFIkbAAAAADgpEjY4DA1Ampo4YMLVSOghtmhAIBLqFFDWrjQWgIAIPEcNjhQoG+gutTvYnYYAOAyAgOlLnxsAgDOwwgbHOZI+hFNXjtZR9KPmB0KALiEI0ekyZOtJQAAkpMkbNOnT1f16tXl4+OjmJgYbdiw4aLtFy1apLp168rHx0cNGzbUt99+a1uWk5OjZ555Rg0bNlTZsmVVqVIl9e7dW4cOHbLr48SJE+rZs6f8/f0VEBCg/v37Kz093SH7d706ePqgRnw/QgdPHzQ7FABwCQcPSiNGWEsAACQnSNg+/fRTDR8+XGPHjtWmTZvUuHFjxcXF6ejRo4W2X7NmjXr06KH+/fvr999/V6dOndSpUyf9/fffkqQzZ85o06ZNeuGFF7Rp0yYtXrxY27dv13333WfXT8+ePbVlyxYtX75cX3/9tX7++WcNHDjQ4fsLAAAAAMVlesI2efJkDRgwQP369VO9evU0Y8YMlSlTRrNmzSq0/dSpU9W2bVuNHDlSUVFRmjBhgpo1a6Zp06ZJksqXL6/ly5era9euqlOnjlq0aKFp06YpISFBSUlJkqTExEQtW7ZMH3zwgWJiYnTrrbfq7bff1oIFCwqMxAEAAACAWUxN2LKzs5WQkKA2bdrY6tzc3NSmTRutXbu20HXWrl1r116S4uLiimwvSampqbJYLAoICLD1ERAQoBtvvNHWpk2bNnJzc9P69esL7SMrK0tpaWl2LwAAAABwJFMTtpSUFOXl5Sk0NNSuPjQ0VMnJyYWuk5ycfFntMzMz9cwzz6hHjx7y9/e39VGxYkW7dh4eHqpQoUKR/cTHx6t8+fK2V0RERLH28XpW3ru87q19r8p7lzc7FABwCeXLS/feay0BAJCc4JJIR8rJyVHXrl1lGIbefffdK+pr1KhRSk1Ntb32799fSlFeu2pWqKmvenylmhVqmh0KALiEmjWlr76ylgAASCY/hy04OFju7u46csH8xUeOHFFYWFih64SFhRWr/blkbd++ffrxxx9to2vn+rhwUpPc3FydOHGiyO16e3vL29u72PsGKScvR6cyTynAJ0Ce7p5mhwMATi8nRzp1SgoIkDz52AQAyOQRNi8vL0VHR2vFihW2uvz8fK1YsUKxsbGFrhMbG2vXXpKWL19u1/5csrZz50798MMPCgoKKtDHqVOnlJCQYKv78ccflZ+fr5iYmNLYNUj66+hfqvhGRf119C+zQwEAl/DXX1LFitYSAADJ5BE2SRo+fLj69OmjG2+8Uc2bN9eUKVOUkZGhfv36SZJ69+6typUrKz4+XpI0bNgwtWzZUpMmTVL79u21YMECbdy4UTNnzpRkTdYefPBBbdq0SV9//bXy8vJs96VVqFBBXl5eioqKUtu2bTVgwADNmDFDOTk5GjJkiLp3765KlSqZcyAAAAAA4AKmJ2zdunXTsWPHNGbMGCUnJ6tJkyZatmyZbWKRpKQkubn9byDw5ptv1vz58/X8889r9OjRioyM1JIlS9SgQQNJ0sGDB/XVV19Jkpo0aWK3rZUrV6pVq1aSpHnz5mnIkCFq3bq13Nzc1LlzZ7311luO32EAAAAAKCbTEzZJGjJkiIYMGVLoslWrVhWo69Kli7p06VJo++rVq8swjEtus0KFCpo/f/5lxQkAAAAAV9M1PUskAAAAALgypxhhw7WpcWhjpT6bqrKeZc0OBQBcQuPGUmqqVJaPTQDA/yNhg8O4u7nL39v/0g0BAJIkd3fJn49NAMB5uCQSDrPz+E7FfRKnncd3mh0KALiEnTuluDhrCQCARMIGBzqdfVrf7/5ep7NPmx0KALiE06el77+3lgAASCRsAAAAAOC0SNgAAAAAwEmRsAEAAACAkyJhg8NE+Edo2j3TFOEfYXYoAOASIiKkadOsJQAAEtP6w4FCyoZocPPBZocBAC4jJEQazMcmAOA8jLDBYU6cPaFP/vxEJ86eMDsUAHAJJ05In3xiLQEAkEjY4EB7T+3Vv774l/ae2mt2KADgEvbulf71L2sJAIBEwgYAAAAATouEDQAAAACcFAkbAAAAADgpEjY4TFnPsmpRpYXKepY1OxQAcAlly0otWlhLAAAkpvWHA9UJrqO1/deaHQYAuIw6daS1fGwCAM7DCBsAAAAAOCkSNjjMpsObZBlv0abDm8wOBQBcwqZNksViLQEAkEjYAAAAAMBpkbABAAAAgJMiYQMAAAAAJ0XCBgAAAABOimn94TD1Qupp5+M7VcW/itmhAIBLqFdP2rlTqsLHJgDg/5GwwWF8PHxUq0Its8MAAJfh4yPV4mMTAHAeEjaUmsTERLv3B88c1Lvb3tW/6/5blctUvqy+goODVbVq1dIMDwCc3p490gsvSBMmSDVqmB0NAMAZkLDhiuWln5QsFvXq1ct+QbikQdLSMUulw5fXp49vGW3flkjSBuC6cvKkNG+eNHw4CRsAwIqEDVcsPytdMgwFdRghz6AIW32O134d1yRrfXbERXqwl3N8v45/PUkpKSkkbAAAALiukbCh1HgGRcg77LybLyzn1RvclAEAAABcLqb1BwAAAAAnRcIGh3E3Kqh8Tg+5GxXMDgUAXEJ4uDR2rLUEAEAqYcL2zz//lHYcuAZ5qIICcnvKQyRsAFAc4eHSuHEkbACA/ylRwlarVi3dcccd+uSTT5SZmVnaMeEaka8zOuuWoHydMTsUAHAJaWnSd99ZSwAApBImbJs2bVKjRo00fPhwhYWFadCgQdqwYUNpxwYXl2M5pKPeY5VjOWR2KADgEnbtktq2tZYAAEglTNiaNGmiqVOn6tChQ5o1a5YOHz6sW2+9VQ0aNNDkyZN17Nix0o4TAAAAAK47VzTpiIeHhx544AEtWrRIr776qnbt2qWnnnpKERER6t27tw4fvsynJQMAAAAAbK4oYdu4caMee+wxhYeHa/LkyXrqqae0e/duLV++XIcOHVLHjh1LK04AAAAAuO6U6MHZkydP1uzZs7V9+3a1a9dOH330kdq1ayc3N2v+V6NGDc2ZM0fVq1cvzVjhYizylEd+uCzyNDsUAHAJ3t5SzZrWEgAAqYQJ27vvvquHH35Yffv2VXgRcw9XrFhRH3744RUFB9fmZVRT5az3zQ4DAFxG/fpMOAIAsFeihG3nzp2XbOPl5aU+ffqUpHsAAAAAgEp4D9vs2bO1aNGiAvWLFi3S3LlzrzgoXBuyLXu03+chZVv2mB0KALiEP/+UQkKsJQAAUgkTtvj4eAUHBxeor1ixol555ZUrDgrXBkN5yrekyVCe2aEAgEvIzZVSUqwlAABSCRO2pKQk1ahRo0B9tWrVlJSUdMVBAQAAAABKmLBVrFhRfxZyvcYff/yhoKCgKw4KAAAAAFDChK1Hjx4aOnSoVq5cqby8POXl5enHH3/UsGHD1L1799KOEQAAAACuSyWaJXLChAnau3evWrduLQ8Paxf5+fnq3bs397DBxtOorLCs1+VpVDY7FABwCbVrS2vWWEsAAKQSJmxeXl769NNPNWHCBP3xxx/y9fVVw4YNVa1atdKODy7MTb7yzo8yOwwAcBnlykmxsWZHAQBwJiVK2M6pXbu2avM1IIqQqxSleX4h/5z75aGCs4oCAOwdOCBNniwNHy5VqWJ2NAAAZ1CihC0vL09z5szRihUrdPToUeXn59st//HHH0slOLi2PMspnfb4UmVz75CHQcIGAJdy9Kj05ptSr14kbAAAqxIlbMOGDdOcOXPUvn17NWjQQBaLpbTjAgAAAIDrXokStgULFmjhwoVq165daccDAAAAAPh/JZrW38vLS7Vq1SrtWAAAAAAA5ylRwjZixAhNnTpVhmGUdjy4hrjLX+Vy28td/maHAgAuIThYeuwxawkAgFTCSyJ//fVXrVy5UkuXLlX9+vXl6elpt3zx4sWlEhxcm4dRUUE5/zY7DABwGVWrStOnmx0FAMCZlChhCwgI0P3331/aseAak69M5VgOyNOoIjf5mB0OADi9M2ekbdukunWlMmXMjgYA4AxKlLDNnj27tOPANSjHckDJPk8oLHOKvA3ueQSAS9m2TYqOlhISpGbNzI4GAOAMSnQPmyTl5ubqhx9+0HvvvafTp09Lkg4dOqT09PRSCw4AAAAArmclGmHbt2+f2rZtq6SkJGVlZemuu+6Sn5+fXn31VWVlZWnGjBmlHScAAAAAXHdKNMI2bNgw3XjjjTp58qR8fX1t9ffff79WrFhRasEBAAAAwPWsRCNsv/zyi9asWSMvLy+7+urVq+vgwYOlEhhcn0Vushi+spT8ylsAuK64uUl+ftYSAACphAlbfn6+8vLyCtQfOHBAfn5+VxwUrg1exg2qmrnI7DAAwGU0aSKlpZkdBQDAmZToO7y7775bU6ZMsb23WCxKT0/X2LFj1a5du9KKDQAAAACuayVK2CZNmqTVq1erXr16yszM1EMPPWS7HPLVV18t7RjhorItSTrk/ZiyLUlmhwIALmHrVql+fWsJAIBUwksiq1Spoj/++EMLFizQn3/+qfT0dPXv3189e/a0m4QE1zdD2cpxS5KhbLNDAQCXkJlpTdYyM82OBADgLEqUsEmSh4eHevXqVZqxAAAAAADOU6JLIj/66KOLvi7H9OnTVb16dfn4+CgmJkYbNmy4aPtFixapbt268vHxUcOGDfXtt9/aLV+8eLHuvvtuBQUFyWKxaPPmzQX6aNWqlSwWi93r0Ucfvay4AQAAAMDRSjTCNmzYMLv3OTk5OnPmjLy8vFSmTBn17t27WP18+umnGj58uGbMmKGYmBhNmTJFcXFx2r59uypWrFig/Zo1a9SjRw/Fx8erQ4cOmj9/vjp16qRNmzapQYMGkqSMjAzdeuut6tq1qwYMGFDktgcMGKAXX3zR9r5MmTLFihkAAAAArpYSjbCdPHnS7pWenq7t27fr1ltv1X/+859i9zN58mQNGDBA/fr1U7169TRjxgyVKVNGs2bNKrT91KlT1bZtW40cOVJRUVGaMGGCmjVrpmnTptna/Otf/9KYMWPUpk2bi267TJkyCgsLs738/f2LHTeKx9MIU0jWC/I0wswOBQBcwg03SF9+aS0BAJBKmLAVJjIyUhMnTiww+laU7OxsJSQk2CVWbm5uatOmjdauXVvoOmvXri2QiMXFxRXZ/mLmzZun4OBgNWjQQKNGjdKZM2cu2j4rK0tpaWl2L1ycm8qpTH6M3FTO7FAAwCUEBEj33WctAQCQSjFhk6wTkRw6dKhYbVNSUpSXl6fQ0FC7+tDQUCUnJxe6TnJy8mW1L8pDDz2kTz75RCtXrtSoUaP08ccfX3IClfj4eJUvX972ioiIuKxtXo/ydFKpHguVp5NmhwIALiE5WYqPt5YAAEglvIftq6++sntvGIYOHz6sadOm6ZZbbimVwBxp4MCBtn83bNhQ4eHhat26tXbv3q2aNWsWus6oUaM0fPhw2/u0tDSStkvItRzXKc+P5JPXTO5GoNnhAIDTO3RIGj1aiouTwriaHACgEiZsnTp1sntvsVgUEhKiO++8U5MmTSpWH8HBwXJ3d9eRI0fs6o8cOaKwIv6XCgsLu6z2xRUTEyNJ2rVrV5EJm7e3t7y9va9oOwAAAABwOUp0SWR+fr7dKy8vT8nJyZo/f77Cw8OL1YeXl5eio6O1YsUKu35XrFih2NjYQteJjY21ay9Jy5cvL7J9cZ2b+r+4sQMAAADA1VDiB2eXhuHDh6tPnz668cYb1bx5c02ZMkUZGRnq16+fJKl3796qXLmy4uPjJVkfJ9CyZUtNmjRJ7du314IFC7Rx40bNnDnT1ueJEyeUlJRku5du+/btkmSbDXL37t2aP3++2rVrp6CgIP3555968skndfvtt6tRo0ZX+QgAAAAAQNFKlLCdfy/XpUyePLnIZd26ddOxY8c0ZswYJScnq0mTJlq2bJltYpGkpCS5uf1vEPDmm2/W/Pnz9fzzz2v06NGKjIzUkiVLbM9gk6z3151L+CSpe/fukqSxY8dq3Lhx8vLy0g8//GBLDiMiItS5c2c9//zzxd4nFI+byqlM3i3MEgkAxRQQID34ILNEAgD+x2IYhnG5K91xxx36/ffflZOTozp16kiSduzYIXd3dzVr1ux/nVss+vHHH0svWieSlpam8uXLKzU11SWf4bZp0yZFR0crrM8UeYfVuqK+0res1PGvJ5VKX5KUlbxLyXOfUEJCgt35BAAAADgjR+YGJRphu/fee+Xn56e5c+cqMNA6+9/JkyfVr18/3XbbbRoxYkSpBgnXZChHeUqVu8rLIk+zwwEAp5edLR09KlWsKHl5mR0NAMAZlGjSkUmTJik+Pt6WrElSYGCgXnrppWLPEolrX7Zlnw769lW2ZZ/ZoQCAS/j7bykiwloCACCVMGFLS0vTsWPHCtQfO3ZMp0+fvuKgAAAAAAAlTNjuv/9+9evXT4sXL9aBAwd04MABff755+rfv78eeOCB0o4RAAAAAK5LJbqHbcaMGXrqqaf00EMPKScnx9qRh4f69++v119/vVQDBAAAAIDrVYkStjJlyuidd97R66+/rt27d0uSatasqbJly5ZqcAAAAABwPbuiB2cfPnxYhw8f1u233y5fX18ZhiGLxVJascHFeRk3qOrZLyS5mx0KALiEJk2kzEzJk4l1AQD/r0T3sB0/flytW7dW7dq11a5dOx0+fFiS1L9/f6b0h41FbrLIU5aSnWYAcN1xc5O8va0lAABSCRO2J598Up6enkpKSlKZMmVs9d26ddOyZctKLTi4thzLQSV7Pascy0GzQwEAl7Bjh9SqlbUEAEAq4SWR33//vb777jtVqVLFrj4yMlL79vHMLVjl66yy3P9Wfs5Zs0MBAJeQni799JO1BABAKuEIW0ZGht3I2jknTpyQt7f3FQcFAAAAAChhwnbbbbfpo48+sr23WCzKz8/Xa6+9pjvuuKPUggMAAACA61mJLol87bXX1Lp1a23cuFHZ2dl6+umntWXLFp04cUKrV68u7RgBAAAA4LpUohG2Bg0aaMeOHbr11lvVsWNHZWRk6IEHHtDvv/+umjVrlnaMcFEeRogqZD8uDyPE7FAAwCVUrSq9/761BABAKsEIW05Ojtq2basZM2boueeec0RMuEa4q7z88uLMDgMAXEZwsPTII2ZHAQBwJpc9wubp6ak///zTEbHgGpOnVJ12/055SjU7FABwCSkp0gcfWEsAAKQSXhLZq1cvffjhh6UdC64xuZZjOuH1tnItx8wOBQBcQlKSNGCAtQQAQCrhpCO5ubmaNWuWfvjhB0VHR6ts2bJ2yydPnlwqwcFeUlKSUkrpa9fExMRS6QcAAACA41xWwvbPP/+oevXq+vvvv9WsWTNJ0o4dO+zaWCyW0osONklJSapTN0qZZ8+YHQoAAACAq+SyErbIyEgdPnxYK1eulCR169ZNb731lkJDQx0SHP4nJSVFmWfPKKjDCHkGRVxxf2f/2ajUXz4phcgAAAAAOMplJWyGYdi9X7p0qTIyMko1IFycZ1CEvMNqXXE/Ocf3l0I0F+cmX3nnNZCbfB2+LQC4FpQrJ7VsaS0BAJBKeA/bORcmcMD5PI3KCsueaHYYAOAyateWVq0yOwoAgDO5rFkiLRZLgXvUuGcNRTGUL0M5MpRvdigA4BLy86WsLGsJAIBUgksi+/btK29vb0lSZmamHn300QKzRC5evLj0IoTLyrb8o2SfJxSWOUXexpVfxgkA17rNm6XoaCkhQfr/ub0AANe5y0rY+vTpY/e+V69epRoMAAAAAOB/Lithmz17tqPiAAAAAABc4LLuYQMAAAAAXD0kbAAAAADgpK5oWn/gYryMaqp8do7cVd7sUADAJTRoIO3fL1WsaHYkAABnQcIGh7HIUx4KNjsMAHAZXl5SlSpmRwEAcCZcEgmHybEk65hXvHIsyWaHAgAu4Z9/pC5drCUAABIJGxwoX+k6475a+Uo3OxQAcAmnTkmffWYtAQCQSNgAAAAAwGmRsAEAAACAkyJhAwAAAAAnRcIGh/EwghSQ01seRpDZoQCAS6hUSXrlFWsJAIDEtP5wIHcFqnxuV7PDAACXERYmjRpldhQAAGfCCBscJl/pOuO2nlkiAaCYTp2SvvqKWSIBAP9DwgaHybEk65j3BJ7DBgDF9M8/UseOPIcNAPA/JGwAAAAA4KRI2AAAAADASZGwAQAAAICTImGDw1jkJc/8qrLIy+xQAMAl+PhI9epZSwAAJKb1hwN5GVVVKesds8MAAJdRr560ZYvZUQAAnAkjbAAAAADgpEjY4DDZln+U5NNF2RbmpwaA4ti8WfL3t5YAAEgkbHAgQ/kyLGdlKN/sUADAJeTnS6dPW0sAACQSNgAAAABwWiRsAAAAAOCkSNgAAAAAwEmRsMFhPI0qCsucIk+jitmhAIBLqFtXSkiwlgAASDyHDQ7kJh95G7XMDgMAXEaZMlKzZmZHAQBwJoywwWFyLUd13PNd5VqOmh0KALiEpCRp8GBrCQCARMIGB8pTmtI9vlGe0swOBQBcQkqK9M471hIAAImEDQAAAACcFgkbAAAAADgpEjYAAAAAcFIkbHAYdyNAfrkd5W4EmB0KALiEihWlJ5+0lgAASEzrDwfyULAq5AwwOwwAcBlVqkiTJ5sdBQDAmTDCBofJ11lluSUqX2fNDgUAXEJ6urR2rbUEAEAiYYMD5VgOKtl7pHIsB80OBQBcwo4d0s03W0sAACQSNgAAAABwWtzDBqeVmJhYan0FBweratWqpdYfAAAAcDWQsMHp5KWflCwW9erVq9T69PEto+3bEknaAAAA4FJMT9imT5+u119/XcnJyWrcuLHefvttNW/evMj2ixYt0gsvvKC9e/cqMjJSr776qtq1a2dbvnjxYs2YMUMJCQk6ceKEfv/9dzVp0sSuj8zMTI0YMUILFixQVlaW4uLi9M477yg0NNRRu3ldsshdboa/LHK/rPXys9Ilw1BQhxHyDIq44jhyju/X8a8nKSUlhYQNgFPz8JCCg60lAACSyQnbp59+quHDh2vGjBmKiYnRlClTFBcXp+3bt6tiIQ+hWbNmjXr06KH4+Hh16NBB8+fPV6dOnbRp0yY1aNBAkpSRkaFbb71VXbt21YABhU8p/+STT+qbb77RokWLVL58eQ0ZMkQPPPCAVq9e7dD9vd54GTUUkTm/xOt7BkXIO6xWKUYEAM6tUSPp2DGzowAAOBNTJx2ZPHmyBgwYoH79+qlevXqaMWOGypQpo1mzZhXafurUqWrbtq1GjhypqKgoTZgwQc2aNdO0adNsbf71r39pzJgxatOmTaF9pKam6sMPP9TkyZN15513Kjo6WrNnz9aaNWu0bt06h+wnAAAAAJSEaQlbdna2EhIS7BIrNzc3tWnTRmvXri10nbVr1xZIxOLi4opsX5iEhATl5OTY9VO3bl1VrVr1ov1kZWUpLS3N7oWLy7bs00HvAcq27DM7FABwCVu2SLVqWUsAACQTE7aUlBTl5eUVuG8sNDRUycnJha6TnJx8We2L6sPLy0sBAQGX1U98fLzKly9ve0VEXPm9Vdc6QznKdTssQzlmhwIALiErS9q921oCACDxHLZiGzVqlFJTU22v/fv3mx0SAAAAgGucaZOOBAcHy93dXUeOHLGrP3LkiMLCwgpdJyws7LLaF9VHdna2Tp06ZTfKdql+vL295e3tXeztAAAAAMCVMm2EzcvLS9HR0VqxYoWtLj8/XytWrFBsbGyh68TGxtq1l6Tly5cX2b4w0dHR8vT0tOtn+/btSkpKuqx+AAAAAMDRTJ3Wf/jw4erTp49uvPFGNW/eXFOmTFFGRob69esnSerdu7cqV66s+Ph4SdKwYcPUsmVLTZo0Se3bt9eCBQu0ceNGzZw509bniRMnlJSUpEOHDkmyJmOSdWQtLCxM5cuXV//+/TV8+HBVqFBB/v7+evzxxxUbG6sWLVpc5SNwbfM0Kqli1nh5GpXMDgUAXEKtWtKyZdYSAADJ5IStW7duOnbsmMaMGaPk5GQ1adJEy5Yts00skpSUJDe3/w0C3nzzzZo/f76ef/55jR49WpGRkVqyZIntGWyS9NVXX9kSPknq3r27JGns2LEaN26cJOnNN9+Um5ubOnfubPfgbJQuN5WRb3602WEAgMvw95fi4syOAgDgTExN2CRpyJAhGjJkSKHLVq1aVaCuS5cu6tKlS5H99e3bV3379r3oNn18fDR9+nRNnz79ckLFZcrVCaV7LFW53HvkoQpmhwMATu/wYem996RBg6TwcLOjAQA4A2aJhMPkWU4o1fM/yrOcMDsUAHAJhw9L48dbSwAAJBI2AAAAAHBaJGwAAAAA4KRI2AAAAADASZGwwWHcVE5lc1vJTeXMDgUAXEJgoNSzp7UEAEByglkice3yNMIUnPOU2WEAgMuoUUP65BOzowAAOBNG2OAwhrKVYzkkQ9lmhwIALiEzU9q1y1oCACCRsMGBsi1JOuQzUNmWJLNDAQCXsHWrFBlpLQEAkEjYAAAAAMBpkbABAAAAgJMiYQMAAAAAJ0XCBgAAAABOimn94TDeRi1VO/u12WEAgMto1kwyDLOjAAA4E0bYAAAAAMBJkbDBYXIsB3TYe4RyLAfMDgUAXML27VJsrLUEAEAiYYMD5StT2W7blS+eAAsAxZGRIa1bZy0BAJBI2AAAAADAaZGwAQAAAICTImEDAAAAACdFwgaH8TBCFZQ9Qh5GqNmhAIBLqF5d+vhjawkAgMRz2OBA7vJTubw7zA4DAFxGhQpSr15mRwEAcCaMsMFh8pSq0+5fK0+pZocCAC7h2DFp+nRrCQCARMIGB8q1HNMJrxnKtfCXBwAUx/790pAh1hIAAImEDQAAAACcFgkbAAAAADgpEjYAAAAAcFIkbHAYN/nKJ6+p3ORrdigA4BL8/KS777aWAABITOsPB/I0Kis0e4LZYQCAy4iMlL77zuwoAADOhBE2OIyhPOXrjAzlmR0KALiEvDwpLc1aAgAgkbDBgbIte7Tft6uyLXvMDgUAXMIff0jly1tLAAAkEjYAAAAAcFokbAAAAADgpEjYAAAAAMBJkbABAAAAgJNiWn84jJdRXVXOzpObypodCgC4hIYNpaNHpYAAsyMBADgLEjY4jEUecld5s8MAAJfh6SmFhJgdBQDAmXBJJBwmx3JYR71eVI7lsNmhAIBL2L1buu8+awkAgETCBgfKV4bOum9QvjLMDgUAXEJqqvTf/1pLAAAkEjYAAAAAcFokbAAAAADgpEjYAAAAAMBJkbDBYTyMIAXm9JeHEWR2KADgEipXliZNspYAAEhM6w8Hcleg/HPvNzsMAHAZoaHS8OFmRwEAcCaMsMFh8pSuDLdflad0s0MBAJdw8qS0aJG1BABAImGDA+VakpXiPVG5lmSzQwEAl7Bnj9S1q7UEAEAiYQMAAAAAp8U9bLhuJCYmllpfwcHBqlq1aqn1BwAAABSGhA3XvLz0k5LFol69epVanz6+ZbR9WyJJGwAAAByKhA0O4yZveeXXlJu8TY0jPytdMgwFdRghz6CIK+4v5/h+Hf96klJSUkjYAJQqX1+paVNrCQCARMIGB/I0IhSeNdXsMGw8gyLkHVbL7DAAoEhRUdKmTWZHAQBwJkw6AgAAAABOioQNDpNt2a19Pp2UbdltdigA4BJ+/13y9raWAABIJGxwIEOGZMm1lgCASzIMKTvbWgIAIJGwAQAAAIDTImEDAAAAACdFwgYAAAAATopp/eEwnkaEwjOny8MIMzsUAHAJUVHS339LN9xgdiQAAGdBwgaHcZO3vIxqZocBAC7D11eqX9/sKAAAzoRLIuEwuZajOu75lnItR80OBQBcwr590iOPWEsAACQSNjhQntKU7vG98pRmdigA4BKOH5c+/NBaAgAgkbABAAAAgNMiYQMAAAAAJ0XCBgAAAABOioQNDuNuBMg/50G5GwFmhwIALiE0VHr2WWsJAIDkJAnb9OnTVb16dfn4+CgmJkYbNmy4aPtFixapbt268vHxUcOGDfXtt9/aLTcMQ2PGjFF4eLh8fX3Vpk0b7dy5065N9erVZbFY7F4TJ04s9X27nnkoWIG5feWhYLNDAQCXULmyFB9vLQEAkJwgYfv00081fPhwjR07Vps2bVLjxo0VFxeno0cLnwp+zZo16tGjh/r376/ff/9dnTp1UqdOnfT333/b2rz22mt66623NGPGDK1fv15ly5ZVXFycMjMz7fp68cUXdfjwYdvr8ccfd+i+Xm/ydUaZbn8qX2fMDgUAXMLp09KqVdYSAADJCRK2yZMna8CAAerXr5/q1aunGTNmqEyZMpo1a1ah7adOnaq2bdtq5MiRioqK0oQJE9SsWTNNmzZNknV0bcqUKXr++efVsWNHNWrUSB999JEOHTqkJUuW2PXl5+ensLAw26ts2bKO3t3rSo7lkI54j1aO5ZDZoQCAS9i5U7rjDmsJAIBkcsKWnZ2thIQEtWnTxlbn5uamNm3aaO3atYWus3btWrv2khQXF2drv2fPHiUnJ9u1KV++vGJiYgr0OXHiRAUFBalp06Z6/fXXlZubW2SsWVlZSktLs3sBAAAAgCN5mLnxlJQU5eXlKfSCu6tDQ0O1bdu2QtdJTk4utH1ycrJt+bm6otpI0tChQ9WsWTNVqFBBa9as0ahRo3T48GFNnjy50O3Gx8dr/Pjxl7eDAAAAAHAFTE3YzDR8+HDbvxs1aiQvLy8NGjRI8fHx8vb2LtB+1KhRduukpaUpIiLiqsQKAAAA4Ppk6iWRwcHBcnd315EjR+zqjxw5orCwsELXCQsLu2j7c+Xl9ClJMTExys3N1d69ewtd7u3tLX9/f7sXLs4iD7kbQbJcv98LAMBl8fS0zhDp6Wl2JAAAZ2Fqwubl5aXo6GitWLHCVpefn68VK1YoNja20HViY2Pt2kvS8uXLbe1r1KihsLAwuzZpaWlav359kX1K0ubNm+Xm5qaKFSteyS7hPF5GdVXJnCsvo7rZoQCAS2jYUDpwwFoCACA5wSWRw4cPV58+fXTjjTeqefPmmjJlijIyMtSvXz9JUu/evVW5cmXFx8dLkoYNG6aWLVtq0qRJat++vRYsWKCNGzdq5syZkiSLxaInnnhCL730kiIjI1WjRg298MILqlSpkjp16iTJOnHJ+vXrdccdd8jPz09r167Vk08+qV69eikwMNCU4wAAAAAAFzI9YevWrZuOHTumMWPGKDk5WU2aNNGyZctsk4YkJSXJze1/A4E333yz5s+fr+eff16jR49WZGSklixZogYNGtjaPP3008rIyNDAgQN16tQp3XrrrVq2bJl8fHwkWS9vXLBggcaNG6esrCzVqFFDTz75pN09arhy2Za9Ouo9VhWzxjPKBgDF8Ndf0j33SEuXMsoGALAyPWGTpCFDhmjIkCGFLlu1alWBui5duqhLly5F9mexWPTiiy/qxRdfLHR5s2bNtG7duhLFiuIzlKs8y3EZKvpxCQCA/8nJkQ4etJYAAEhO8OBsAAAAAEDhSNgAAAAAwEmRsAEAAACAkyJhg8N4GpUUmvWKPI1KZocCAC4hMlJaudJaAgAgOcmkI7g2uamMfPIbmR2GwyQmJpZaX8HBwapatWqp9QfANfn5Sa1amR0FAMCZkLDBYXKVotMeX8svt4M8FGx2OKUmL/2kZLGoV69epdanj28Zbd+WSNIGXOcOHpSmTZOGDJEqVzY7GgCAMyBhg8PkWU4pzfMzlcm7VR7GtZOw5WelS4ahoA4j5BkUccX95Rzfr+NfT1JKSgoJG3CdO3JEmjhR6tKFhA0AYEXCBpSQZ1CEvMNqmR0GAAAArmFMOgIAAAAAToqEDQAAAACcFAkbHMZd/iqXe7fc5W92KADgEoKCpP79rSUAABL3sMGBPIyKCsoZanYYAOAyqlWTPvjA7CgAAM6EETY4TL6ylG3Zp3xlmR0KALiEs2elLVusJQAAEgkbHCjHsl+HfQYrx7Lf7FAAwCUkJkoNGlhLAAAkEjYAAAAAcFokbAAAAADgpEjYAAAAAMBJkbDBYSyySIaHtQQAXJLFInl5WUsAACSm9YcDeRk1VS1zidlhAIDLaNpUymJiXQDAeRhhAwAAAAAnRcIGh8mx7Ndh72FM6w8AxZSYKDVrxrT+AID/IWGDw+QrS9luu3lwNgAU09mz0u+/8+BsAMD/cA8b4CQSS/Er9eDgYFWtWrXU+gMAAIA5SNgAk+Wln5QsFvXq1avU+vTxLaPt2xJJ2gAAAFwcCRtgsvysdMkwFNRhhDyDIq64v5zj+3X860lKSUkhYQMAAHBxJGxwGA8jTMFZz8rDCDM7FJfgGRQh77BaZocBwEQ1akgLF1pLAAAkEjY4kLvKqWz+rWaHAQAuIzBQ6tLF7CgAAM6EWSLhMHk6qTSPL5Snk2aHAgAu4cgRafJkawkAgETCBgfKtRzXSc8PlWs5bnYoAOASDh6URoywlgAASCRsAAAAAOC0SNgAAAAAwEmRsAEAAACAkyJhg8O4qax885rLTWXNDgUAXEL58tK991pLAAAkpvWHA3ka4aqYPcbsMADAZdSsKX31ldlRAACcCSNscBhDucpTqgzlmh0KALiEnBzp2DFrCQCARMIGB8q27NUB357Ktuw1OxQAcAl//SVVrGgtAQCQuCQSuGYlJiaWSj/BwcGqWrVqqfQFAACAy0PCBlxj8tJPShaLevXqVSr9+fiW0fZtiSRtAAAAJiBhA64x+VnpkmEoqMMIeQZFXFFfOcf36/jXk5SSkkLCBgAAYAISNuAa5RkUIe+wWmaHAQAAgCtAwgaH8TJqKOLsQlnkbXYoAOASGjeWUlOlsjy+EgDw/0jY4DAWucuiMmaHAQAuw91d8vc3OwoAgDNhWn84TI7loI54vaAcy0GzQwEAl7BzpxQXZy0BAJAYYYMD5eusMt1/V37OWbNDwRUqrUcESDwmALiY06el77+3lgAASCRsAC6itB8RIPGYAAAAgMtBwgagSKX5iACJxwQAAABcLhI2AJfEIwIAAADMwaQjcBgPI0QVsh+VhxFidigA4BIiIqRp06wlAAASI2xwIHeVl19eB7PDAACXERIiDR5sdhQAAGfCCBscJk+nle6+UnliujMAKI4TJ6RPPrGWAABIjLDBgXItR3Tca5LCMqfI3fAzOxw4ER4TABRu717pX/+SEhKkChXMjgYA4AxI2ABcNTwmAAAA4PKQsAG4ahz1mIBffvlFUVFRpRAhI3YAAMC5kLABuOpK6zEBjNgBAIBrHQkbHMZNPvLKryM3+ZgdCq5RPNgb15qyZaUWLawlAAASCRscyNOoovCsSWaHgesAD/bGtaJOHWntWrOjAAA4E6b1BwAAAAAnxQgbHCbLskvJPk8oLHOKvA1GP+A6eOwAzLJpkxQdbZ3Wv1kzs6MBADgDEjYA+H+OmMTE29tHn3/+mcLDw0ulPxJAAACuLyRsAPD/SnsSk8wDW3Tqxw/UoUOHUojOilksAQC4vpCwAcAFSmsSk5zj+5nFEgAAXBESNgBwsNKexbK07rHj8koAAJwfCRscxsuoqkqZM+VhBJsdCnBNKO177Jz98sqkpCSlpKSUWn9ZWVny9vYutf4ckfDWqyft3ClVqVKq3QIAXBgJGxzGIi95GpXMDgO4ZpTmPXbnLq/85ZdfFBUVVSrxlWZCdPjwYXV+sIuyMs+WSn+SJIubZOSXWneOSHh9fKRaTKoLADgPCRscJseSrFSPT1Q+t5c8jTCzwwGuGaVxiaUjZsQs7YRIUqnd/3f2n41K/eUTp7+fcM8e6YUXpAkTpBo1Sq1bAIALc4qEbfr06Xr99deVnJysxo0b6+2331bz5s2LbL9o0SK98MIL2rt3ryIjI/Xqq6+qXbt2tuWGYWjs2LF6//33derUKd1yyy169913FRkZaWtz4sQJPf744/rvf/8rNzc3de7cWVOnTlW5cuUcuq/Xk3ylK8NjlfxyO5kdCoALlPaMmKWdEJ3rr1QngJHz3k94bnQyMdFX8+ZF6Z57EnXyZMlHF13h8k8AQPGYnrB9+umnGj58uGbMmKGYmBhNmTJFcXFx2r59uypWrFig/Zo1a9SjRw/Fx8erQ4cOmj9/vjp16qRNmzapQYMGkqTXXntNb731lubOnasaNWrohRdeUFxcnLZu3SofHx9JUs+ePXX48GEtX75cOTk56tevnwYOHKj58+df1f0HADM5a0J0rj9nVeojlLbRyaaSNqlXr56Sfi+F/kpHaV/+Wdr3J5Z2Qlma8ZHsArhSpidskydP1oABA9SvXz9J0owZM/TNN99o1qxZevbZZwu0nzp1qtq2bauRI0dKkiZMmKDly5dr2rRpmjFjhgzD0JQpU/T888+rY8eOkqSPPvpIoaGhWrJkibp3767ExEQtW7ZMv/32m2688UZJ0ttvv6127drpjTfeUKVK3HcFAChaaY5Qnj86KTXT8a+loA5PyTPo2BX3V5qXf5bW/Y6OuD+xNB9QX9rxlWZsUumPntKfc/TlCv058xcjzr6vV8rUhC07O1sJCQkaNWqUrc7NzU1t2rTR2rVrC11n7dq1Gj58uF1dXFyclixZIknas2ePkpOT1aZNG9vy8uXLKyYmRmvXrlX37t21du1aBQQE2JI1SWrTpo3c3Ny0fv163X///QW2m5WVpaysLNv71NRUSVJaWtrl73gJpKenW+NI3qX87Mwr7u/ct9el0V9RfeV4HZDCpazDl7eN0oyN/hzzs6U/1+/PmWNzpf7yc7KuuD8jN9vWl3RGUpryc86UuN/z+yuNfc09bf2DqlTvd5Tkf9MDci8fcsX95Bzbq/Q/vivVB9RLpROfY2KzSDLozyn6c+bYSr8/L28fffKxdRDkSh05ckS9/tVb2VlX/hllZf6+ZmRkSLLemlXqDBMdPHjQkGSsWbPGrn7kyJFG8+bNC13H09PTmD9/vl3d9OnTjYoVKxqGYRirV682JBmHDh2ya9OlSxeja9euhmEYxssvv2zUrl27QN8hISHGO++8U+h2x44da8h6JvDixYsXL168ePHixYtXgdfu3buLlwhdBtMviXQVo0aNshvZy8/P14kTJxQUFCSLxXJFfaelpSkiIkL79++Xv7//lYaKYuK4m4djbw6Ou3k49ubguJuHY28Ojrt5UlNTVbVqVVWoUKHU+zY1YQsODpa7u7uOHDliV3/kyBGFhRU+DXxYWNhF258rjxw5Yne9+JEjR9SkSRNbm6NHj9r1kZubqxMnThS5XW9v7wLXxgYEBFx8By+Tv78/v1wm4Libh2NvDo67eTj25uC4m4djbw6Ou3nc3NxKv89S7/EyeHl5KTo6WitWrLDV5efna8WKFYqNjS10ndjYWLv2krR8+XJb+xo1aigsLMyuTVpamtavX29rExsbq1OnTikhIcHW5scff1R+fr5iYmJKbf8AAAAA4EqYfknk8OHD1adPH914441q3ry5pkyZooyMDNuskb1791blypUVHx8vSRo2bJhatmypSZMmqX379lqwYIE2btyomTNnSpIsFoueeOIJvfTSS4qMjLRN61+pUiV16tRJkhQVFaW2bdtqwIABmjFjhnJycjRkyBB1796dGSIBAAAAOA3TE7Zu3brp2LFjGjNmjJKTk9WkSRMtW7bMNitLUlKS3dDizTffrPnz5+v555/X6NGjFRkZqSVLltiewSZJTz/9tDIyMjRw4ECdOnVKt956q5YtW2Z7BpskzZs3T0OGDFHr1q1tD85+6623rt6On8fb21tjx44t1elIcWkcd/Nw7M3BcTcPx94cHHfzcOzNwXE3jyOPvcUwHDH3JAAAAADgSpl6DxsAAAAAoGgkbAAAAADgpEjYAAAAAMBJkbABAAAAgJMiYTPZ9OnTVb16dfn4+CgmJkYbNmwwO6Rrzrhx42SxWOxedevWtS3PzMzU4MGDFRQUpHLlyqlz584FHs6OS/v555917733qlKlSrJYLFqyZIndcsMwNGbMGIWHh8vX11dt2rTRzp077dqcOHFCPXv2lL+/vwICAtS/f3+lp6dfxb1wTZc69n379i3wO9C2bVu7Nhz7yxcfH6+bbrpJfn5+qlixojp16qTt27fbtSnO50tSUpLat2+vMmXKqGLFiho5cqRyc3Ov5q64lOIc91atWhU45x999FG7Nhz3y/fuu++qUaNGtocyx8bGaunSpbblnO+Ocanjzvl+dUycONH2+LBzrtY5T8Jmok8//VTDhw/X2LFjtWnTJjVu3FhxcXE6evSo2aFdc+rXr6/Dhw/bXr/++qtt2ZNPPqn//ve/WrRokX766ScdOnRIDzzwgInRuqaMjAw1btxY06dPL3T5a6+9prfeekszZszQ+vXrVbZsWcXFxSkzM9PWpmfPntqyZYuWL1+ur7/+Wj///LMGDhx4tXbBZV3q2EtS27Zt7X4H/vOf/9gt59hfvp9++kmDBw/WunXrtHz5cuXk5Ojuu+9WRkaGrc2lPl/y8vLUvn17ZWdna82aNZo7d67mzJmjMWPGmLFLLqE4x12SBgwYYHfOv/baa7ZlHPeSqVKliiZOnKiEhARt3LhRd955pzp27KgtW7ZI4nx3lEsdd4nz3dF+++03vffee2rUqJFd/VU75w2Ypnnz5sbgwYNt7/Py8oxKlSoZ8fHxJkZ17Rk7dqzRuHHjQpedOnXK8PT0NBYtWmSrS0xMNCQZa9euvUoRXnskGV988YXtfX5+vhEWFma8/vrrtrpTp04Z3t7exn/+8x/DMAxj69athiTjt99+s7VZunSpYbFYjIMHD1612F3dhcfeMAyjT58+RseOHYtch2NfOo4ePWpIMn766SfDMIr3+fLtt98abm5uRnJysq3Nu+++a/j7+xtZWVlXdwdc1IXH3TAMo2XLlsawYcOKXIfjXnoCAwONDz74gPP9Kjt33A2D893RTp8+bURGRhrLly+3O9ZX85xnhM0k2dnZSkhIUJs2bWx1bm5uatOmjdauXWtiZNemnTt3qlKlSrrhhhvUs2dPJSUlSZISEhKUk5Nj93OoW7euqlatys+hFO3Zs0fJycl2x7l8+fKKiYmxHee1a9cqICBAN954o61NmzZt5ObmpvXr11/1mK81q1atUsWKFVWnTh39+9//1vHjx23LOPalIzU1VZJUoUIFScX7fFm7dq0aNmyo0NBQW5u4uDilpaXZfXuOol143M+ZN2+egoOD1aBBA40aNUpnzpyxLeO4X7m8vDwtWLBAGRkZio2N5Xy/Si487udwvjvO4MGD1b59e7tzW7q6n/EeV7gPKKGUlBTl5eXZ/QAlKTQ0VNu2bTMpqmtTTEyM5syZozp16ujw4cMaP368brvtNv39999KTk6Wl5eXAgIC7NYJDQ1VcnKyOQFfg84dy8LO93PLkpOTVbFiRbvlHh4eqlChAj+LK9S2bVs98MADqlGjhnbv3q3Ro0frnnvu0dq1a+Xu7s6xLwX5+fl64okndMstt6hBgwaSVKzPl+Tk5EJ/L84tw8UVdtwl6aGHHlK1atVUqVIl/fnnn3rmmWe0fft2LV68WBLH/Ur89ddfio2NVWZmpsqVK6cvvvhC9erV0+bNmznfHaio4y5xvjvSggULtGnTJv32228Fll3Nz3gSNlzz7rnnHtu/GzVqpJiYGFWrVk0LFy6Ur6+viZEBV0f37t1t/27YsKEaNWqkmjVratWqVWrdurWJkV07Bg8erL///tvu/lg4XlHH/fz7Lxs2bKjw8HC1bt1au3fvVs2aNa92mNeUOnXqaPPmzUpNTdVnn32mPn366KeffjI7rGteUce9Xr16nO8Osn//fg0bNkzLly+Xj4+PqbFwSaRJgoOD5e7uXmAmmSNHjigsLMykqK4PAQEBql27tnbt2qWwsDBlZ2fr1KlTdm34OZSuc8fyYud7WFhYgQl3cv+vvXsPiqp+/wD+XlzBZQFBQVgVcQ0veGlFvIR8x3sIFqPVKEOGiKKIgCBoQuMF08KsNHUQ0wwoK03LtLxfFrJ1UUG5KW6CBF4wJsyEQHLZ5/eHw5mO3EQQsN/zmtkZzuec8znP5wL4+DnnoNfj7t27PBYtrE+fPrC2tkZeXh4A7vvmCgkJwU8//QS1Wo2ePXsK5U/y88XOzq7O74uafax+9fV7XUaNGgUAojnP/f50jI2N4ejoCBcXF8TGxkKlUmHTpk0835+x+vq9LjzfW0Z6ejpKSkowbNgwSKVSSKVSpKSkYPPmzZBKpbC1tW21Oc8JWxsxNjaGi4sLTp06JZQZDAacOnVKdE8ya3nl5eXIz8+HQqGAi4sLOnbsKBoHnU6HoqIiHocWpFQqYWdnJ+rn+/fv49y5c0I/u7q64t69e0hPTxeOOX36NAwGg/DLh7WMmzdvorS0FAqFAgD3/dMiIoSEhGD//v04ffo0lEqlaP+T/HxxdXVFdna2KGE+ceIELCwshNudmFhj/V6XjIwMABDNee73lmEwGFBVVcXzvZXV9HtdeL63jIkTJyI7OxsZGRnCZ/jw4Zg5c6bwdavN+ZZ4ewp7Ort37yYTExNKTEykK1eu0Pz588nS0lL0JhnWfJGRkZScnEwFBQWk0Who0qRJZG1tTSUlJUREtGDBAurVqxedPn2a0tLSyNXVlVxdXds46udPWVkZXbp0iS5dukQAaMOGDXTp0iUqLCwkIqJ169aRpaUlHThwgLKysmjq1KmkVCqpsrJSqMPDw4OcnZ3p3Llz9Msvv1Dfvn3Jx8enrZr03Gio78vKymjJkiWk1WqpoKCATp48ScOGDaO+ffvSgwcPhDq475suKCiIOnfuTMnJyVRcXCx8KioqhGMa+/mi1+tp8ODB5O7uThkZGXT06FGysbGh6OjotmjSc6Gxfs/Ly6N3332X0tLSqKCggA4cOEB9+vShMWPGCHVwvz+dqKgoSklJoYKCAsrKyqKoqCiSSCR0/PhxIuL5/qw01O8831vX42/kbK05zwlbG9uyZQv16tWLjI2NaeTIkZSamtrWIf3neHt7k0KhIGNjY+rRowd5e3tTXl6esL+yspIWLlxIVlZWZGpqSq+99hoVFxe3YcTPJ7VaTQBqffz8/Ijo0av9V6xYQba2tmRiYkITJ04knU4nqqO0tJR8fHzIzMyMLCwsyN/fn8rKytqgNc+Xhvq+oqKC3N3dycbGhjp27EgODg40b968Wv8xxH3fdHX1OQBKSEgQjnmSny+//fYbeXp6kkwmI2tra4qMjKSHDx+2cmueH431e1FREY0ZM4a6dOlCJiYm5OjoSEuXLqW//vpLVA/3e9PNmTOHHBwcyNjYmGxsbGjixIlCskbE8/1Zaajfeb63rscTttaa8xIioiavETLGGGOMMcYYe+b4GTbGGGOMMcYYa6c4YWOMMcYYY4yxdooTNsYYY4wxxhhrpzhhY4wxxhhjjLF2ihM2xhhjjDHGGGunOGFjjDHGGGOMsXaKEzbGGGOMMcYYa6c4YWOMMcYYY4yxdooTNsYYaydmz56NadOmtXUYzaLRaDBkyBB07NjxuW1L79698cknn7R4vTExMRg6dGiz60lMTISlpWWz63kSOp0OdnZ2KCsra5XrPStNGdOXXnoJ33333bMNiDHGmoATNsYYawUSiaTBT0xMDDZt2oTExMRWjy05OVkUi0wmw6BBg7B9+/Ym1xUREYGhQ4eioKCgTdqyY8cOqFQqmJmZwdLSEs7OzoiNjW31OOqyZMkSnDp1qtn1eHt749dff22BiBoXHR2N0NBQmJubt8r12oPly5cjKioKBoOhrUNhjDEAgLStA2CMsf8PiouLha/37NmDlStXQqfTCWVmZmYwMzNri9AEOp0OFhYWqKysxI8//oigoCC88MILmDhx4hPXkZ+fjwULFqBnz55PHcc///wDY2PjJp/3+eefIzw8HJs3b8bYsWNRVVWFrKws5OTkPHUsLamlxlgmk0Emk7VARA0rKirCTz/9hC1btjzza7Unnp6eCAgIwJEjR/DKK6+0dTiMMcYrbIwx1hrs7OyET+fOnSGRSERlZmZmtW6JHDduHEJDQxEeHg4rKyvY2tpix44d+Pvvv+Hv7w9zc3M4OjriyJEjomvl5OTA09MTZmZmsLW1ha+vL/74449GY+zWrRvs7OygVCqxaNEiKJVKXLx4UdhvMBgQGxsLpVIJmUwGlUqFffv2AQB+++03SCQSlJaWYs6cOZBIJMIKW0pKCkaOHAkTExMoFApERUVBr9eL2hkSEoLw8HBYW1tj8uTJT9WOgwcPYsaMGZg7dy4cHR0xaNAg+Pj44L333hNdKzw8XHTetGnTMHv2bFFZWVkZfHx8IJfL0aNHD8TFxYn2SyQSfPrpp3j11VdhamoKJycnaLVa5OXlYdy4cZDL5Rg9ejTy8/OFcx6/JTI5ORkjR46EXC6HpaUl3NzcUFhYCADIzMzE+PHjYW5uDgsLC7i4uCAtLQ1A3bdExsfH44UXXoCxsTH69++PL7/8sla8n332GV577TWYmpqib9++OHjwYL19CQDffvstVCoVevToIZQVFhbCy8sLVlZWkMvlGDRoEA4fPizsb2zMDAYD1q9fD0dHR5iYmKBXr16i8cnOzsaECRMgk8nQtWtXzJ8/H+Xl5cL+mu+Rjz76CAqFAl27dkVwcDAePnwoHFNSUgIvLy/IZDIolUp89dVXonYREWJiYtCrVy+YmJige/fuWLRokbC/Q4cOmDJlCnbv3t1g/zDGWGvhhI0xxtqxpKQkWFtb4/z58wgNDUVQUBCmT5+O0aNH4+LFi3B3d4evry8qKioAAPfu3cOECRPg7OyMtLQ0HD16FL///jtmzJjxxNckIhw9ehRFRUUYNWqUUB4bG4svvvgC27Ztw+XLl7F48WK89dZbSElJgb29PYqLi2FhYYFPPvkExcXF8Pb2xq1btzBlyhSMGDECmZmZiI+Px86dO7F27dpa7TQ2NoZGo8G2bdueqh12dnZITU0Vkp7m+PDDD6FSqXDp0iVERUUhLCwMJ06cEB2zZs0azJo1CxkZGRgwYADefPNNBAYGIjo6GmlpaSAihISE1Fm/Xq/HtGnTMHbsWGRlZUGr1WL+/PmQSCQAgJkzZ6Jnz564cOEC0tPTERUVhY4dO9ZZ1/79+xEWFobIyEjk5OQgMDAQ/v7+UKvVouNWr16NGTNmICsrC1OmTMHMmTNx9+7devvgzJkzGD58uKgsODgYVVVV+Pnnn5GdnY0PPvhAWDV8kjGLjo7GunXrsGLFCly5cgVff/01bG1tAQB///03Jk+eDCsrK1y4cAF79+7FyZMna/WhWq1Gfn4+1Go1kpKSkJiYKLr9dvbs2bhx4wbUajX27duHrVu3oqSkRNj/3XffYePGjfj0009x7do1/PDDDxgyZIjoGiNHjsSZM2fq7RvGGGtVxBhjrFUlJCRQ586da5X7+fnR1KlThe2xY8fS//73P2Fbr9eTXC4nX19foay4uJgAkFarJSKiNWvWkLu7u6jeGzduEADS6XR1xqNWqwkAyeVyksvlJJVKycjIiNauXSsc8+DBAzI1NaWzZ8+Kzp07dy75+PgI2507d6aEhARh+5133qH+/fuTwWAQyuLi4sjMzIyqq6uFdjo7O4vqfZp23L59m1566SUCQP369SM/Pz/as2ePcJ2aa4WFhYnOmzp1Kvn5+QnbDg4O5OHhITrG29ubPD09hW0AtHz5cmFbq9USANq5c6dQ9s0331CnTp2E7VWrVpFKpSIiotLSUgJAycnJdbbF3NycEhMT69z3+PwZPXo0zZs3T3TM9OnTacqUKfXGW15eTgDoyJEjdV6DiEilUtG7774rKhsyZAjFxMTUeXxjY3b//n0yMTGhHTt21Hn+9u3bycrKisrLy4WyQ4cOkZGREd25c4eIHn2PODg4kF6vF7XV29ubiIh0Oh0BoPPnzwv7c3NzCQBt3LiRiIg+/vhj6tevH/3zzz/1tv3AgQNkZGQkmjuMMdZWeIWNMcbasRdffFH4ukOHDujatatoNaBmdaJmBSEzMxNqtVp4XsrMzAwDBgwAANHteXU5c+YMMjIykJGRgc8++wzvv/8+4uPjAQB5eXmoqKjAyy+/LKr7iy++aLDe3NxcuLq6CitHAODm5oby8nLcvHlTKHNxcRGd9zTtUCgU0Gq1yM7ORlhYGPR6Pfz8/ODh4dHkF0i4urrW2s7NzRWV/Xtsasbh8bF58OAB7t+/X6v+Ll26YPbs2Zg8eTK8vLywadMm0XOOERERCAgIwKRJk7Bu3bpG+9jNzU1U5ubm1mC8crkcFhYWopWnx1VWVqJTp06iskWLFmHt2rVwc3PDqlWrkJWVJexrbMxyc3NRVVVV7zORubm5UKlUkMvlonYYDAbR856DBg1Chw4dhG2FQiG0Izc3F1KpVDSfBgwYILqFdPr06aisrESfPn0wb9487N+/X3SLLvDoOUGDwYCqqqp6+4cxxloLJ2yMMdaOPX4bnEQiEZXVJEI1CUl5eTm8vLyExKvmc+3aNYwZM6bBaymVSuHZL39/f/j6+grPF9U8R3To0CFRvVeuXBGeY2uOf/8jvbntGDx4MBYuXIhdu3bhxIkTOHHiBFJSUgAARkZGICLR8f9+/qkp6hqHhsbmcQkJCdBqtRg9ejT27NmDfv36ITU1FcCj590uX76MV155BadPn8bAgQOxf//+p4qzrnhr4msokbW2tsaff/4pKgsICMD169fh6+uL7OxsDB8+XHgpSWNj1lIvSmlqOx5nb28PnU6HrVu3QiaTYeHChRgzZoxoHty9exdyubxVXu7CGGON4YSNMcb+Q4YNG4bLly+jd+/ecHR0FH0eT4oa06FDB1RWVgIABg4cCBMTExQVFdWq197evt46al7G8e8kSaPRwNzcvME3SbZUOwYOHAjg0fNRAGBjYyNayaqurq7zLZI1idO/t52cnJ74uk/K2dkZ0dHROHv2LAYPHoyvv/5a2NevXz8sXrwYx48fx+uvv46EhIQ663BycoJGoxGVaTQaoe3Nie3KlSu1yu3t7bFgwQJ8//33iIyMxI4dOwA0PmZ9+/aFTCar908bODk5ITMzUxirmnYYGRmhf//+TxTzgAEDoNfrkZ6eLpTpdDrcu3dPdJxMJoOXlxc2b96M5ORkYWW2Rk5ODpydnZ/omowx9qxxwsYYY/8hwcHBuHv3Lnx8fHDhwgXk5+fj2LFj8Pf3R3V1dYPnlpSU4M6dOygsLMTevXvx5ZdfYurUqQAAc3NzLFmyBIsXL0ZSUhLy8/Nx8eJFbNmyBUlJSfXWuXDhQty4cQOhoaG4evUqDhw4gFWrViEiIgJGRvX/CnqadgQFBWHNmjXQaDQoLCxEamoqZs2aBRsbG+EWxwkTJuDQoUM4dOgQrl69iqCgoFr/mAceJQrr16/Hr7/+iri4OOzduxdhYWEN9l9TFBQUIDo6GlqtFoWFhTh+/DiuXbsGJycnVFZWIiQkBMnJySgsLIRGo8GFCxfqTRiXLl2KxMRExMfH49q1a9iwYQO+//57LFmypFkxTp48GVqtVtTf4eHhOHbsGAoKCnDx4kWo1WohrsbGrFOnTli2bBnefvtt4Vba1NRU7Ny5E8CjF6106tQJfn5+yMnJgVqtRmhoKHx9fYVbThvTv39/eHh4IDAwEOfOnUN6ejoCAgJEK2WJiYnYuXMncnJycP36dezatQsymQwODg7CMWfOnIG7u3uz+o8xxloK/x02xhj7D+nevTs0Gg2WLVsGd3d3VFVVwcHBAR4eHg0mSACEVQypVAp7e3sEBgYiJiZG2L9mzRrY2NggNjYW169fh6WlJYYNG4Z33nmn3jp79OiBw4cPY+nSpVCpVOjSpQvmzp2L5cuXt3g7Jk2ahM8//xzx8fEoLS2FtbU1XF1dcerUKXTt2hUAMGfOHGRmZmLWrFmQSqVYvHgxxo8fX6uuyMhIpKWlYfXq1bCwsMCGDRuEPzfQEkxNTXH16lUkJSWhtLQUCoUCwcHBCAwMhF6vR2lpKWbNmoXff/8d1tbWeP3117F69eo665o2bRo2bdqEjz76CGFhYVAqlUhISMC4ceOaFaOnpyekUilOnjwptL26uhrBwcG4efMmLCws4OHhgY0bNwJ4sjFbsWIFpFIpVq5cidu3b0OhUGDBggVCnxw7dgxhYWEYMWIETE1N8cYbb2DDhg1NijshIQEBAQEYO3YsbG1tsXbtWqxYsULYb2lpiXXr1iEiIgLV1dUYMmQIfvzxR2GO3Lp1C2fPnsWuXbua1X+MMdZSJPT4zfyMMcYYYwDi4uJw8OBBHDt2rK1DaTXLli3Dn3/+ie3bt7d1KIwxBoBX2BhjjDFWj8DAQNy7dw9lZWUwNzdv63BaRbdu3RAREdHWYTDGmIBX2BhjjDHGGGOsneKXjjDGGGOMMcZYO8UJG2OMMcYYY4y1U5ywMcYYY4wxxlg7xQkbY4wxxhhjjLVTnLAxxhhjjDHGWDvFCRtjjDHGGGOMtVOcsDHGGGOMMcZYO8UJG2OMMcYYY4y1U5ywMcYYY4wxxlg79X9nyzBh1NwgYwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0.7946706482372897"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the mean and standard deviation\n",
    "mean_value = df['time_before_submission_seconds'].mean()\n",
    "std_dev = df['time_before_submission_seconds'].std()\n",
    "\n",
    "# Define outlier thresholds\n",
    "lower_bound = mean_value - 2 * std_dev\n",
    "upper_bound = mean_value + 2 * std_dev\n",
    "\n",
    "# Calculate the median value\n",
    "median_value = df['time_before_submission_seconds'].median()\n",
    "\n",
    "#Replace outliers with the median value\n",
    "df['time_before_submission_seconds'] = np.where(\n",
    "    (df['time_before_submission_seconds'] < lower_bound) | (df['time_before_submission_seconds'] > upper_bound),\n",
    "    median_value,\n",
    "    df['time_before_submission_seconds']\n",
    ")\n",
    "\n",
    "# df['time_before_submission_seconds'] = np.where(\n",
    "#     (df['time_before_submission_seconds']>60*15),\n",
    "#     median_value,\n",
    "#     df['time_before_submission_seconds']\n",
    "# )\n",
    "# Plot the histogram\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.hist(df['time_before_submission_seconds'], bins=500, edgecolor='black', density=True)\n",
    "plt.axvline(lower_bound, color='red', linestyle='dashed', linewidth=1, label='Lower bound')\n",
    "plt.axvline(upper_bound, color='red', linestyle='dashed', linewidth=1, label='Upper bound')\n",
    "plt.axvline(mean_value, color='blue', linestyle='dashed', linewidth=1, label='Mean')\n",
    "plt.axvline(median_value, color='green', linestyle='dashed', linewidth=1, label='Median')\n",
    "plt.title('Histogram of Time Before Submission (seconds)')\n",
    "plt.xlabel('Time Before Submission (seconds)')\n",
    "plt.ylabel('Frequency')\n",
    "plt.xlim(-10, 400)\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "df['time_before_submission_seconds'].mean()/60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "94442a85-1ce3-43ff-850e-79e170493bb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47.680238894237384"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "0.7946706482372897*60"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "884d3ce6-4c9c-4bb5-be32-569c61f84d7a",
   "metadata": {},
   "source": [
    "# Identify outliers using the IQR method\n",
    "Q1 = df['time_before_submission_seconds'].quantile(0.25)\n",
    "Q3 = df['time_before_submission_seconds'].quantile(0.75)\n",
    "IQR = Q3 - Q1\n",
    "\n",
    "# Define outlier thresholds\n",
    "lower_bound = Q1 - 1.5 * IQR\n",
    "upper_bound = Q3 + 1.5 * IQR\n",
    "\n",
    "# Replace outliers with the median value\n",
    "df['time_before_submission_seconds'] = np.where(\n",
    "    (df['time_before_submission_seconds'] < lower_bound) | (df['time_before_submission_seconds'] > upper_bound),\n",
    "    median_value,\n",
    "    df['time_before_submission_seconds']\n",
    ")\n",
    "\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.hist(df['time_before_submission_seconds'], bins=500, edgecolor='black', density=True)\n",
    "plt.axvline(lower_bound, color='red', linestyle='dashed', linewidth=1, label='Lower bound')\n",
    "plt.axvline(upper_bound, color='red', linestyle='dashed', linewidth=1, label='Upper bound')\n",
    "plt.axvline(mean_value, color='blue', linestyle='dashed', linewidth=1, label='Mean')\n",
    "plt.axvline(median_value, color='green', linestyle='dashed', linewidth=1, label='Median')\n",
    "plt.title('Histogram of Time Before Submission (seconds)')\n",
    "plt.xlabel('Time Before Submission (seconds)')\n",
    "plt.ylabel('Frequency')\n",
    "plt.xlim(-1, 200)\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "df['time_before_submission_seconds'].mean()/60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1f1bbf74-46dc-461e-888f-b8242f3cb3c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "96.30083805555554"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minutes = df['time_before_submission_seconds'].sum()/60\n",
    "hours = minutes/60\n",
    "hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1744a7f3-0de9-42cc-b6f5-ffc54f54d444",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15771.871830204806 -15361.161136765115\n"
     ]
    }
   ],
   "source": [
    "print(upper_bound, lower_bound)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a0e5cd80-e8c2-4b55-9321-32c4a96f719b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>classification_id</th>\n",
       "      <th>user_name</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_ip</th>\n",
       "      <th>workflow_id</th>\n",
       "      <th>workflow_name</th>\n",
       "      <th>workflow_version</th>\n",
       "      <th>created_at</th>\n",
       "      <th>gold_standard</th>\n",
       "      <th>expert</th>\n",
       "      <th>metadata</th>\n",
       "      <th>annotations</th>\n",
       "      <th>subject_data</th>\n",
       "      <th>subject_ids</th>\n",
       "      <th>started_at</th>\n",
       "      <th>finished_at</th>\n",
       "      <th>time_before_submission</th>\n",
       "      <th>time_before_submission_seconds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4798</th>\n",
       "      <td>526544790</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>dfc5b8148f8ee749e72e</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-03 17:59:21 UTC</td>\n",
       "      <td>True</td>\n",
       "      <td>owner</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"777a08164cd7f8e01a2...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"94323602\":{\"retired\":null,\"Filename\":\"llama_...</td>\n",
       "      <td>94323602</td>\n",
       "      <td>2023-12-03 17:58:00.350000+00:00</td>\n",
       "      <td>2023-12-03 17:59:20.297000+00:00</td>\n",
       "      <td>0 days 00:01:19.947000</td>\n",
       "      <td>79.947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4799</th>\n",
       "      <td>526686902</td>\n",
       "      <td>not-logged-in-8d8a50a5ab4ff2a4c782</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8d8a50a5ab4ff2a4c782</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:46:57 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"6a2fe65aa1413a34ff5...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"94323999\":{\"retired\":null,\"Filename\":\"mammot...</td>\n",
       "      <td>94323999</td>\n",
       "      <td>2023-12-04 12:44:02.301000+00:00</td>\n",
       "      <td>2023-12-04 12:46:53.021000+00:00</td>\n",
       "      <td>0 days 00:02:50.720000</td>\n",
       "      <td>170.720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4800</th>\n",
       "      <td>526687176</td>\n",
       "      <td>not-logged-in-329ddf4b8909579aafd1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>329ddf4b8909579aafd1</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:49:34 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"6a2fe65aa1413a34ff5...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"94323887\":{\"retired\":null,\"Filename\":\"llema_...</td>\n",
       "      <td>94323887</td>\n",
       "      <td>2023-12-04 12:47:04.270000+00:00</td>\n",
       "      <td>2023-12-04 12:49:29.649000+00:00</td>\n",
       "      <td>0 days 00:02:25.379000</td>\n",
       "      <td>145.379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4801</th>\n",
       "      <td>526687284</td>\n",
       "      <td>not-logged-in-ab81d6eefed8fef91f6b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ab81d6eefed8fef91f6b</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:50:58 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"6a2fe65aa1413a34ff5...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"94323658\":{\"retired\":null,\"Filename\":\"llama_...</td>\n",
       "      <td>94323658</td>\n",
       "      <td>2023-12-04 12:49:34.509000+00:00</td>\n",
       "      <td>2023-12-04 12:50:50.005000+00:00</td>\n",
       "      <td>0 days 00:01:15.496000</td>\n",
       "      <td>75.496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4802</th>\n",
       "      <td>526687346</td>\n",
       "      <td>not-logged-in-02c614ded7364b0426f2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02c614ded7364b0426f2</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:51:38 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"6a2fe65aa1413a34ff5...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"94323967\":{\"retired\":null,\"Filename\":\"mammot...</td>\n",
       "      <td>94323967</td>\n",
       "      <td>2023-12-04 12:50:53.835000+00:00</td>\n",
       "      <td>2023-12-04 12:51:36.455000+00:00</td>\n",
       "      <td>0 days 00:00:42.620000</td>\n",
       "      <td>42.620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7284</th>\n",
       "      <td>569847408</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>2a0244cc976d3aca5b3f</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:03 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262312\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262312</td>\n",
       "      <td>2024-06-27 18:25:15.663000+00:00</td>\n",
       "      <td>2024-06-27 18:26:03.753000+00:00</td>\n",
       "      <td>0 days 00:00:48.090000</td>\n",
       "      <td>48.090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7285</th>\n",
       "      <td>569847516</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:24 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262399\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262399</td>\n",
       "      <td>2024-06-27 18:26:04.605000+00:00</td>\n",
       "      <td>2024-06-27 18:26:23.770000+00:00</td>\n",
       "      <td>0 days 00:00:19.165000</td>\n",
       "      <td>19.165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7286</th>\n",
       "      <td>569847621</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:42 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262362\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262362</td>\n",
       "      <td>2024-06-27 18:26:24.544000+00:00</td>\n",
       "      <td>2024-06-27 18:26:42.526000+00:00</td>\n",
       "      <td>0 days 00:00:17.982000</td>\n",
       "      <td>17.982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7287</th>\n",
       "      <td>569847743</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:01 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262329\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262329</td>\n",
       "      <td>2024-06-27 18:26:43.577000+00:00</td>\n",
       "      <td>2024-06-27 18:27:01.197000+00:00</td>\n",
       "      <td>0 days 00:00:17.620000</td>\n",
       "      <td>17.620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7288</th>\n",
       "      <td>569848012</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:43 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...</td>\n",
       "      <td>[{\"task\":\"T0\",\"task_label\":\"Is this question s...</td>\n",
       "      <td>{\"100262438\":{\"retired\":null,\"Filename\":\"egsm_...</td>\n",
       "      <td>100262438</td>\n",
       "      <td>2024-06-27 18:27:03.278000+00:00</td>\n",
       "      <td>2024-06-27 18:27:43.852000+00:00</td>\n",
       "      <td>0 days 00:00:40.574000</td>\n",
       "      <td>40.574</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2491 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      classification_id                           user_name    user_id  \\\n",
       "4798          526544790                         bryanchrist  2631956.0   \n",
       "4799          526686902  not-logged-in-8d8a50a5ab4ff2a4c782        NaN   \n",
       "4800          526687176  not-logged-in-329ddf4b8909579aafd1        NaN   \n",
       "4801          526687284  not-logged-in-ab81d6eefed8fef91f6b        NaN   \n",
       "4802          526687346  not-logged-in-02c614ded7364b0426f2        NaN   \n",
       "...                 ...                                 ...        ...   \n",
       "7284          569847408                         bryanchrist  2631956.0   \n",
       "7285          569847516                         bryanchrist  2631956.0   \n",
       "7286          569847621                         bryanchrist  2631956.0   \n",
       "7287          569847743                         bryanchrist  2631956.0   \n",
       "7288          569848012                         bryanchrist  2631956.0   \n",
       "\n",
       "                   user_ip  workflow_id       workflow_name  workflow_version  \\\n",
       "4798  dfc5b8148f8ee749e72e        24740  Classify Questions           139.206   \n",
       "4799  8d8a50a5ab4ff2a4c782        24740  Classify Questions           139.206   \n",
       "4800  329ddf4b8909579aafd1        24740  Classify Questions           139.206   \n",
       "4801  ab81d6eefed8fef91f6b        24740  Classify Questions           139.206   \n",
       "4802  02c614ded7364b0426f2        24740  Classify Questions           139.206   \n",
       "...                    ...          ...                 ...               ...   \n",
       "7284  2a0244cc976d3aca5b3f        24740  Classify Questions           142.214   \n",
       "7285  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "7286  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "7287  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "7288  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "\n",
       "                   created_at gold_standard expert  \\\n",
       "4798  2023-12-03 17:59:21 UTC          True  owner   \n",
       "4799  2023-12-04 12:46:57 UTC           NaN    NaN   \n",
       "4800  2023-12-04 12:49:34 UTC           NaN    NaN   \n",
       "4801  2023-12-04 12:50:58 UTC           NaN    NaN   \n",
       "4802  2023-12-04 12:51:38 UTC           NaN    NaN   \n",
       "...                       ...           ...    ...   \n",
       "7284  2024-06-27 18:26:03 UTC           NaN    NaN   \n",
       "7285  2024-06-27 18:26:24 UTC           NaN    NaN   \n",
       "7286  2024-06-27 18:26:42 UTC           NaN    NaN   \n",
       "7287  2024-06-27 18:27:01 UTC           NaN    NaN   \n",
       "7288  2024-06-27 18:27:43 UTC           NaN    NaN   \n",
       "\n",
       "                                               metadata  \\\n",
       "4798  {\"source\":\"api\",\"session\":\"777a08164cd7f8e01a2...   \n",
       "4799  {\"source\":\"api\",\"session\":\"6a2fe65aa1413a34ff5...   \n",
       "4800  {\"source\":\"api\",\"session\":\"6a2fe65aa1413a34ff5...   \n",
       "4801  {\"source\":\"api\",\"session\":\"6a2fe65aa1413a34ff5...   \n",
       "4802  {\"source\":\"api\",\"session\":\"6a2fe65aa1413a34ff5...   \n",
       "...                                                 ...   \n",
       "7284  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7285  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7286  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7287  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "7288  {\"source\":\"api\",\"session\":\"79416595f7756f3a0bd...   \n",
       "\n",
       "                                            annotations  \\\n",
       "4798  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "4799  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "4800  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "4801  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "4802  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "...                                                 ...   \n",
       "7284  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7285  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7286  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7287  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "7288  [{\"task\":\"T0\",\"task_label\":\"Is this question s...   \n",
       "\n",
       "                                           subject_data  subject_ids  \\\n",
       "4798  {\"94323602\":{\"retired\":null,\"Filename\":\"llama_...     94323602   \n",
       "4799  {\"94323999\":{\"retired\":null,\"Filename\":\"mammot...     94323999   \n",
       "4800  {\"94323887\":{\"retired\":null,\"Filename\":\"llema_...     94323887   \n",
       "4801  {\"94323658\":{\"retired\":null,\"Filename\":\"llama_...     94323658   \n",
       "4802  {\"94323967\":{\"retired\":null,\"Filename\":\"mammot...     94323967   \n",
       "...                                                 ...          ...   \n",
       "7284  {\"100262312\":{\"retired\":null,\"Filename\":\"egsm_...    100262312   \n",
       "7285  {\"100262399\":{\"retired\":null,\"Filename\":\"egsm_...    100262399   \n",
       "7286  {\"100262362\":{\"retired\":null,\"Filename\":\"egsm_...    100262362   \n",
       "7287  {\"100262329\":{\"retired\":null,\"Filename\":\"egsm_...    100262329   \n",
       "7288  {\"100262438\":{\"retired\":null,\"Filename\":\"egsm_...    100262438   \n",
       "\n",
       "                           started_at                      finished_at  \\\n",
       "4798 2023-12-03 17:58:00.350000+00:00 2023-12-03 17:59:20.297000+00:00   \n",
       "4799 2023-12-04 12:44:02.301000+00:00 2023-12-04 12:46:53.021000+00:00   \n",
       "4800 2023-12-04 12:47:04.270000+00:00 2023-12-04 12:49:29.649000+00:00   \n",
       "4801 2023-12-04 12:49:34.509000+00:00 2023-12-04 12:50:50.005000+00:00   \n",
       "4802 2023-12-04 12:50:53.835000+00:00 2023-12-04 12:51:36.455000+00:00   \n",
       "...                               ...                              ...   \n",
       "7284 2024-06-27 18:25:15.663000+00:00 2024-06-27 18:26:03.753000+00:00   \n",
       "7285 2024-06-27 18:26:04.605000+00:00 2024-06-27 18:26:23.770000+00:00   \n",
       "7286 2024-06-27 18:26:24.544000+00:00 2024-06-27 18:26:42.526000+00:00   \n",
       "7287 2024-06-27 18:26:43.577000+00:00 2024-06-27 18:27:01.197000+00:00   \n",
       "7288 2024-06-27 18:27:03.278000+00:00 2024-06-27 18:27:43.852000+00:00   \n",
       "\n",
       "     time_before_submission  time_before_submission_seconds  \n",
       "4798 0 days 00:01:19.947000                          79.947  \n",
       "4799 0 days 00:02:50.720000                         170.720  \n",
       "4800 0 days 00:02:25.379000                         145.379  \n",
       "4801 0 days 00:01:15.496000                          75.496  \n",
       "4802 0 days 00:00:42.620000                          42.620  \n",
       "...                     ...                             ...  \n",
       "7284 0 days 00:00:48.090000                          48.090  \n",
       "7285 0 days 00:00:19.165000                          19.165  \n",
       "7286 0 days 00:00:17.982000                          17.982  \n",
       "7287 0 days 00:00:17.620000                          17.620  \n",
       "7288 0 days 00:00:40.574000                          40.574  \n",
       "\n",
       "[2491 rows x 18 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(0, len(df)):\n",
    "    my_json = json.loads(df.iloc[i]['annotations'])\n",
    "    try:\n",
    "        if my_json[2]['task_label']=='What math operations/concepts are required to solve the problem? Please select all that apply. ':\n",
    "            start_index = i\n",
    "            break\n",
    "    except:\n",
    "        pass\n",
    "df = df[start_index:]\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86730697-6f13-485b-b0ee-0d253ec35f76",
   "metadata": {},
   "source": [
    "## Create Appropriateness, Solvability, Accuracy, Topic, Math Operation and Notes Columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bfd23bfb-f20c-4cbd-a81b-550170b68c2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'task': 'T0',\n",
       "  'task_label': 'Is this question solvable? A solvable question means that it can be solved with the information present. ',\n",
       "  'value': 'Yes'},\n",
       " {'task': 'T4',\n",
       "  'task_label': 'Is the solution for this question correct?',\n",
       "  'value': 'Yes'},\n",
       " {'task': 'T6',\n",
       "  'task_label': 'What math operations/concepts are required to solve the problem? Please select all that apply. ',\n",
       "  'value': ['Subtraction', 'Addition']},\n",
       " {'task': 'T1',\n",
       "  'task_label': 'Would you feel comfortable giving this question to a student? Although questions have various difficulties, a middle school student should be able to solve them all.',\n",
       "  'value': 'Yes: This question is understandable, appropriate and a middle school student or younger could solve it '},\n",
       " {'task': 'T5',\n",
       "  'task_label': 'Does the question incorporate the given topic? ',\n",
       "  'value': 'Yes'},\n",
       " {'task': 'T3',\n",
       "  'value': '',\n",
       "  'task_label': 'This question is optional. If you have any feedback about this particular question, you can tell us here. '}]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_json = json.loads(df.iloc[0]['annotations'])\n",
    "my_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3f92a754-4743-4227-9f9c-43aa1ed72d58",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/2447154829.py:27: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['solvability']=solvability\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/2447154829.py:28: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['appropriateness']=appropriateness\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/2447154829.py:29: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['accuracy']=accuracy\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/2447154829.py:30: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['operations'] = ops\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/2447154829.py:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['notes']=notes\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/2447154829.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['topic']=topic\n"
     ]
    }
   ],
   "source": [
    "solvability = []\n",
    "accuracy = []\n",
    "appropriateness = []\n",
    "topic = []\n",
    "ops = []\n",
    "notes = []\n",
    "for i in range(0, len(df)):\n",
    "    my_json = json.loads(df.iloc[i]['annotations'])\n",
    "    solvability.append(my_json[0]['value'])\n",
    "    if my_json[1]['task']=='T4':\n",
    "        accuracy.append(my_json[1]['value'])\n",
    "    if my_json[1]['task']!='T4':\n",
    "        accuracy.append(\"NA\")\n",
    "        appropriateness.append('NA')\n",
    "        ops.append('NA')\n",
    "        topic.append(my_json[1]['value'])\n",
    "    if my_json[2]['task']=='T6':\n",
    "        ops.append(my_json[2]['value'])\n",
    "    if my_json[2]['task']!='T6':\n",
    "        notes.append(my_json[2]['value'])\n",
    "    try:\n",
    "        appropriateness.append(my_json[3]['value'])\n",
    "        topic.append(my_json[4]['value'])\n",
    "        notes.append(my_json[5]['value'])\n",
    "    except:\n",
    "        pass\n",
    "df['solvability']=solvability\n",
    "df['appropriateness']=appropriateness\n",
    "df['accuracy']=accuracy\n",
    "df['operations'] = ops\n",
    "df['notes']=notes\n",
    "df['topic']=topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "03583e6d-bcb5-48d7-987f-2fab9081e0da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_name\n",
       "bryanchrist                           2005\n",
       "kidleader                              149\n",
       "Raceaturtle                            135\n",
       "not-logged-in-d686d4334d00d113492a      23\n",
       "not-logged-in-8a7e887ccc7df8e54aff      18\n",
       "not-logged-in-a995833e0064b1a62f8a      16\n",
       "not-logged-in-21dd56b989228e883dee      12\n",
       "not-logged-in-01d1a4de2519268b1480       9\n",
       "not-logged-in-54d52b93eaf28770678a       8\n",
       "not-logged-in-32c214cfab64a34bb50f       7\n",
       "not-logged-in-c87d545dfc280e8ea529       7\n",
       "not-logged-in-2627d903c7ae22fddac0       6\n",
       "not-logged-in-a639c542b133d230733d       6\n",
       "not-logged-in-d43d96c64ff9f31ea237       5\n",
       "not-logged-in-c89b428b819bcea962db       5\n",
       "not-logged-in-f6e35db41dff1f893344       5\n",
       "not-logged-in-16c3a563e4df628d4ce6       5\n",
       "not-logged-in-695a0b07f6c9a0239266       5\n",
       "not-logged-in-1177246b7a4084f7c5af       5\n",
       "not-logged-in-91ae611983ee4e790f42       5\n",
       "not-logged-in-4d1e22248ff8d5ac8526       4\n",
       "cate11                                   4\n",
       "not-logged-in-419b4c84ee0ddc826ec3       4\n",
       "not-logged-in-295dea9e2f5b8feca6a9       4\n",
       "not-logged-in-4434f482a6b3f09f0b29       4\n",
       "not-logged-in-207d334a2eae31af9301       3\n",
       "not-logged-in-26b3f83f9405b275bde3       3\n",
       "not-logged-in-254f1b0adc11b05dfe9c       2\n",
       "not-logged-in-9cfcf496ebf1b813f19b       2\n",
       "not-logged-in-56a3d371332e2b539a25       2\n",
       "not-logged-in-3d368448749e6fc7f07d       1\n",
       "not-logged-in-5a847c15a3bd5c1659fe       1\n",
       "not-logged-in-726cbb6c1fb6ea8719ad       1\n",
       "not-logged-in-329ddf4b8909579aafd1       1\n",
       "not-logged-in-263b2fed971713715cec       1\n",
       "not-logged-in-95af40fb4ea11242ed23       1\n",
       "not-logged-in-ab81d6eefed8fef91f6b       1\n",
       "not-logged-in-02c614ded7364b0426f2       1\n",
       "not-logged-in-14a4e4cc131dfa6862b1       1\n",
       "not-logged-in-1f62abf888da440851e4       1\n",
       "not-logged-in-fb027728d9aacb480a0d       1\n",
       "not-logged-in-5f8bdba5be391d2962a3       1\n",
       "not-logged-in-80cd4372150b2f3e7a80       1\n",
       "not-logged-in-ed28403e8e1a346cff41       1\n",
       "not-logged-in-1d91c43026bd00c462ab       1\n",
       "not-logged-in-bdc2477f3a5c3e6096c4       1\n",
       "not-logged-in-e8a2b4e0a2577d5a1dbd       1\n",
       "not-logged-in-c81757e666dc6bd0b749       1\n",
       "not-logged-in-8ed8477c6dd7b3ad4a76       1\n",
       "not-logged-in-0f1f8588405680b90eee       1\n",
       "not-logged-in-af0b3f211a07cd9fa3f5       1\n",
       "not-logged-in-8d8a50a5ab4ff2a4c782       1\n",
       "not-logged-in-64f847afc8766ebae3cf       1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['user_name'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97ff5142-8371-4174-bf83-2ef6a89e97e8",
   "metadata": {},
   "source": [
    "df = df[df['user_name']=='bryanchrist']\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cf69aa9b-f5fc-4a34-8164-ec4c9e4df892",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>classification_id</th>\n",
       "      <th>user_name</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_ip</th>\n",
       "      <th>workflow_id</th>\n",
       "      <th>workflow_name</th>\n",
       "      <th>workflow_version</th>\n",
       "      <th>created_at</th>\n",
       "      <th>gold_standard</th>\n",
       "      <th>expert</th>\n",
       "      <th>...</th>\n",
       "      <th>started_at</th>\n",
       "      <th>finished_at</th>\n",
       "      <th>time_before_submission</th>\n",
       "      <th>time_before_submission_seconds</th>\n",
       "      <th>solvability</th>\n",
       "      <th>appropriateness</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>operations</th>\n",
       "      <th>notes</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4799</th>\n",
       "      <td>526686902</td>\n",
       "      <td>not-logged-in-8d8a50a5ab4ff2a4c782</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8d8a50a5ab4ff2a4c782</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:46:57 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2023-12-04 12:44:02.301000+00:00</td>\n",
       "      <td>2023-12-04 12:46:53.021000+00:00</td>\n",
       "      <td>0 days 00:02:50.720000</td>\n",
       "      <td>170.720</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes: This question is understandable, appropri...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>[Subtraction, Addition]</td>\n",
       "      <td>The question is too simplistic</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4800</th>\n",
       "      <td>526687176</td>\n",
       "      <td>not-logged-in-329ddf4b8909579aafd1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>329ddf4b8909579aafd1</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:49:34 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2023-12-04 12:47:04.270000+00:00</td>\n",
       "      <td>2023-12-04 12:49:29.649000+00:00</td>\n",
       "      <td>0 days 00:02:25.379000</td>\n",
       "      <td>145.379</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No: This question contains distracting typos o...</td>\n",
       "      <td>No</td>\n",
       "      <td>[Division]</td>\n",
       "      <td>There is a contradiction in the question</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4805</th>\n",
       "      <td>526687683</td>\n",
       "      <td>not-logged-in-5a847c15a3bd5c1659fe</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5a847c15a3bd5c1659fe</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:55:30 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2023-12-04 12:54:14.652000+00:00</td>\n",
       "      <td>2023-12-04 12:55:25.270000+00:00</td>\n",
       "      <td>0 days 00:01:10.618000</td>\n",
       "      <td>70.618</td>\n",
       "      <td>No</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>The meaning of this question is unclear</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5713</th>\n",
       "      <td>528390771</td>\n",
       "      <td>not-logged-in-207d334a2eae31af9301</td>\n",
       "      <td>NaN</td>\n",
       "      <td>207d334a2eae31af9301</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2023-12-11 13:43:37 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2023-12-11 13:41:59.394000+00:00</td>\n",
       "      <td>2023-12-11 13:43:37.525000+00:00</td>\n",
       "      <td>0 days 00:01:38.131000</td>\n",
       "      <td>98.131</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes: This question is understandable, appropri...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>[Subtraction]</td>\n",
       "      <td>The extra steps 2-4 are unnecessary and may ca...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5716</th>\n",
       "      <td>528391151</td>\n",
       "      <td>not-logged-in-ed28403e8e1a346cff41</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ed28403e8e1a346cff41</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2023-12-11 13:46:02 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2023-12-11 13:44:28.621000+00:00</td>\n",
       "      <td>2023-12-11 13:46:02.234000+00:00</td>\n",
       "      <td>0 days 00:01:33.613000</td>\n",
       "      <td>93.613</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes: This question is understandable, appropri...</td>\n",
       "      <td>No</td>\n",
       "      <td>[Division, Multiplication]</td>\n",
       "      <td>The lift 2000 pounds is superfluous</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6637</th>\n",
       "      <td>550976754</td>\n",
       "      <td>Raceaturtle</td>\n",
       "      <td>2668246.0</td>\n",
       "      <td>addfa2d90650c7f7f330</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-03-28 14:18:32 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-28 14:17:09.305000+00:00</td>\n",
       "      <td>2024-03-28 14:18:32.098000+00:00</td>\n",
       "      <td>0 days 00:01:22.793000</td>\n",
       "      <td>82.793</td>\n",
       "      <td>No</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>You need all the raw numbers to calculate aver...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6638</th>\n",
       "      <td>550976927</td>\n",
       "      <td>Raceaturtle</td>\n",
       "      <td>2668246.0</td>\n",
       "      <td>addfa2d90650c7f7f330</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-03-28 14:19:30 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-28 14:18:33.186000+00:00</td>\n",
       "      <td>2024-03-28 14:19:30.682000+00:00</td>\n",
       "      <td>0 days 00:00:57.496000</td>\n",
       "      <td>57.496</td>\n",
       "      <td>No</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>Numbers given don’t add up</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6643</th>\n",
       "      <td>550977726</td>\n",
       "      <td>Raceaturtle</td>\n",
       "      <td>2668246.0</td>\n",
       "      <td>c700b041b49cb026c6c1</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-03-28 14:24:21 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-28 14:23:14.701000+00:00</td>\n",
       "      <td>2024-03-28 14:24:21.701000+00:00</td>\n",
       "      <td>0 days 00:01:07</td>\n",
       "      <td>67.000</td>\n",
       "      <td>No</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>Unclear if the q is asking area or perimeter</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6798</th>\n",
       "      <td>551078982</td>\n",
       "      <td>Raceaturtle</td>\n",
       "      <td>2668246.0</td>\n",
       "      <td>5865c0e8fe405ea8cfd2</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-03-28 21:15:53 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-28 21:14:29.524000+00:00</td>\n",
       "      <td>2024-03-28 21:15:53.056000+00:00</td>\n",
       "      <td>0 days 00:01:23.532000</td>\n",
       "      <td>83.532</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes: This question is understandable, appropri...</td>\n",
       "      <td>No</td>\n",
       "      <td>[Fractions]</td>\n",
       "      <td>8</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6809</th>\n",
       "      <td>551080827</td>\n",
       "      <td>Raceaturtle</td>\n",
       "      <td>2668246.0</td>\n",
       "      <td>5865c0e8fe405ea8cfd2</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-03-28 21:24:15 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2024-03-28 21:23:12.589000+00:00</td>\n",
       "      <td>2024-03-28 21:24:15.454000+00:00</td>\n",
       "      <td>0 days 00:01:02.865000</td>\n",
       "      <td>62.865</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No: This question is strange, confusing, conta...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>[Subtraction]</td>\n",
       "      <td>Clarify what black or white is</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>79 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      classification_id                           user_name    user_id  \\\n",
       "4799          526686902  not-logged-in-8d8a50a5ab4ff2a4c782        NaN   \n",
       "4800          526687176  not-logged-in-329ddf4b8909579aafd1        NaN   \n",
       "4805          526687683  not-logged-in-5a847c15a3bd5c1659fe        NaN   \n",
       "5713          528390771  not-logged-in-207d334a2eae31af9301        NaN   \n",
       "5716          528391151  not-logged-in-ed28403e8e1a346cff41        NaN   \n",
       "...                 ...                                 ...        ...   \n",
       "6637          550976754                         Raceaturtle  2668246.0   \n",
       "6638          550976927                         Raceaturtle  2668246.0   \n",
       "6643          550977726                         Raceaturtle  2668246.0   \n",
       "6798          551078982                         Raceaturtle  2668246.0   \n",
       "6809          551080827                         Raceaturtle  2668246.0   \n",
       "\n",
       "                   user_ip  workflow_id       workflow_name  workflow_version  \\\n",
       "4799  8d8a50a5ab4ff2a4c782        24740  Classify Questions           139.206   \n",
       "4800  329ddf4b8909579aafd1        24740  Classify Questions           139.206   \n",
       "4805  5a847c15a3bd5c1659fe        24740  Classify Questions           139.206   \n",
       "5713  207d334a2eae31af9301        24740  Classify Questions           142.214   \n",
       "5716  ed28403e8e1a346cff41        24740  Classify Questions           142.214   \n",
       "...                    ...          ...                 ...               ...   \n",
       "6637  addfa2d90650c7f7f330        24740  Classify Questions           142.214   \n",
       "6638  addfa2d90650c7f7f330        24740  Classify Questions           142.214   \n",
       "6643  c700b041b49cb026c6c1        24740  Classify Questions           142.214   \n",
       "6798  5865c0e8fe405ea8cfd2        24740  Classify Questions           142.214   \n",
       "6809  5865c0e8fe405ea8cfd2        24740  Classify Questions           142.214   \n",
       "\n",
       "                   created_at gold_standard expert  ...  \\\n",
       "4799  2023-12-04 12:46:57 UTC           NaN    NaN  ...   \n",
       "4800  2023-12-04 12:49:34 UTC           NaN    NaN  ...   \n",
       "4805  2023-12-04 12:55:30 UTC           NaN    NaN  ...   \n",
       "5713  2023-12-11 13:43:37 UTC           NaN    NaN  ...   \n",
       "5716  2023-12-11 13:46:02 UTC           NaN    NaN  ...   \n",
       "...                       ...           ...    ...  ...   \n",
       "6637  2024-03-28 14:18:32 UTC           NaN    NaN  ...   \n",
       "6638  2024-03-28 14:19:30 UTC           NaN    NaN  ...   \n",
       "6643  2024-03-28 14:24:21 UTC           NaN    NaN  ...   \n",
       "6798  2024-03-28 21:15:53 UTC           NaN    NaN  ...   \n",
       "6809  2024-03-28 21:24:15 UTC           NaN    NaN  ...   \n",
       "\n",
       "                           started_at                      finished_at  \\\n",
       "4799 2023-12-04 12:44:02.301000+00:00 2023-12-04 12:46:53.021000+00:00   \n",
       "4800 2023-12-04 12:47:04.270000+00:00 2023-12-04 12:49:29.649000+00:00   \n",
       "4805 2023-12-04 12:54:14.652000+00:00 2023-12-04 12:55:25.270000+00:00   \n",
       "5713 2023-12-11 13:41:59.394000+00:00 2023-12-11 13:43:37.525000+00:00   \n",
       "5716 2023-12-11 13:44:28.621000+00:00 2023-12-11 13:46:02.234000+00:00   \n",
       "...                               ...                              ...   \n",
       "6637 2024-03-28 14:17:09.305000+00:00 2024-03-28 14:18:32.098000+00:00   \n",
       "6638 2024-03-28 14:18:33.186000+00:00 2024-03-28 14:19:30.682000+00:00   \n",
       "6643 2024-03-28 14:23:14.701000+00:00 2024-03-28 14:24:21.701000+00:00   \n",
       "6798 2024-03-28 21:14:29.524000+00:00 2024-03-28 21:15:53.056000+00:00   \n",
       "6809 2024-03-28 21:23:12.589000+00:00 2024-03-28 21:24:15.454000+00:00   \n",
       "\n",
       "     time_before_submission  time_before_submission_seconds solvability  \\\n",
       "4799 0 days 00:02:50.720000                         170.720         Yes   \n",
       "4800 0 days 00:02:25.379000                         145.379         Yes   \n",
       "4805 0 days 00:01:10.618000                          70.618          No   \n",
       "5713 0 days 00:01:38.131000                          98.131         Yes   \n",
       "5716 0 days 00:01:33.613000                          93.613         Yes   \n",
       "...                     ...                             ...         ...   \n",
       "6637 0 days 00:01:22.793000                          82.793          No   \n",
       "6638 0 days 00:00:57.496000                          57.496          No   \n",
       "6643        0 days 00:01:07                          67.000          No   \n",
       "6798 0 days 00:01:23.532000                          83.532         Yes   \n",
       "6809 0 days 00:01:02.865000                          62.865         Yes   \n",
       "\n",
       "                                        appropriateness accuracy  \\\n",
       "4799  Yes: This question is understandable, appropri...      Yes   \n",
       "4800  No: This question contains distracting typos o...       No   \n",
       "4805                                                 NA       NA   \n",
       "5713  Yes: This question is understandable, appropri...      Yes   \n",
       "5716  Yes: This question is understandable, appropri...       No   \n",
       "...                                                 ...      ...   \n",
       "6637                                                 NA       NA   \n",
       "6638                                                 NA       NA   \n",
       "6643                                                 NA       NA   \n",
       "6798  Yes: This question is understandable, appropri...       No   \n",
       "6809  No: This question is strange, confusing, conta...      Yes   \n",
       "\n",
       "                      operations  \\\n",
       "4799     [Subtraction, Addition]   \n",
       "4800                  [Division]   \n",
       "4805                          NA   \n",
       "5713               [Subtraction]   \n",
       "5716  [Division, Multiplication]   \n",
       "...                          ...   \n",
       "6637                          NA   \n",
       "6638                          NA   \n",
       "6643                          NA   \n",
       "6798                 [Fractions]   \n",
       "6809               [Subtraction]   \n",
       "\n",
       "                                                  notes topic  \n",
       "4799                     The question is too simplistic   Yes  \n",
       "4800           There is a contradiction in the question   Yes  \n",
       "4805            The meaning of this question is unclear    No  \n",
       "5713  The extra steps 2-4 are unnecessary and may ca...   Yes  \n",
       "5716                The lift 2000 pounds is superfluous   Yes  \n",
       "...                                                 ...   ...  \n",
       "6637  You need all the raw numbers to calculate aver...   Yes  \n",
       "6638                        Numbers given don’t add up    Yes  \n",
       "6643       Unclear if the q is asking area or perimeter   Yes  \n",
       "6798                                                  8   Yes  \n",
       "6809                     Clarify what black or white is   Yes  \n",
       "\n",
       "[79 rows x 24 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp = df[df['notes']!=\"\"]\n",
    "temp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e58d11e-3495-4257-9f55-ddc1b4509b1b",
   "metadata": {},
   "source": [
    "## Assign Question Numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "56be1055-8a80-499c-8658-54f1cc1061c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/107389611.py:35: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['question_number'] = question_numbers\n"
     ]
    }
   ],
   "source": [
    "question_numbers = []\n",
    "model = []\n",
    "\n",
    "for i in range(len(df)):\n",
    "    my_json = json.loads(df.iloc[i]['subject_data'])\n",
    "    subject_id = df.iloc[i]['subject_ids']\n",
    "    question_number = my_json[f'{subject_id}']['Filename']\n",
    "    question_number = question_number.split(\".txt\")[0]\n",
    "    question_numbers.append(question_number)\n",
    "\n",
    "    if 'llema' in question_number:\n",
    "        model.append('llema')\n",
    "    elif 'llama' in question_number and 'llama_gsm' not in question_number:\n",
    "        model.append('llama')    \n",
    "    elif 'mammoth' in question_number:\n",
    "        model.append('mammoth')\n",
    "    elif 'mathwell' in question_number:\n",
    "        model.append('mathwell')\n",
    "    elif 'gpt35' in question_number: \n",
    "        model.append('gpt35')\n",
    "\n",
    "    elif 'gpt4' in question_number: \n",
    "        model.append('gpt4')\n",
    "\n",
    "    elif \"llama_gsm\" in question_number:\n",
    "        model.append(\"llama_gsm\")\n",
    "\n",
    "    elif \"egsm_only\" in question_number:\n",
    "        model.append(\"egsm_only\")\n",
    "    else:\n",
    "        # Append None if none of the models are found in question_number\n",
    "        model.append(None)\n",
    "\n",
    "# Create a DataFrame with question_numbers and model lists\n",
    "df['question_number'] = question_numbers\n",
    "new_df = pd.DataFrame({'question_number': question_numbers, 'model': model})\n",
    "new_df = new_df.drop_duplicates()\n",
    "# Filter out rows where model is not None\n",
    "filtered_df = new_df[new_df['model'].notna()]\n",
    "\n",
    "# Merge the filtered DataFrame with the original DataFrame df\n",
    "df = pd.merge(df, filtered_df, on='question_number', how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "30cf5349-e669-4aaf-ae7b-a0b80c14df87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>classification_id</th>\n",
       "      <th>user_name</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_ip</th>\n",
       "      <th>workflow_id</th>\n",
       "      <th>workflow_name</th>\n",
       "      <th>workflow_version</th>\n",
       "      <th>created_at</th>\n",
       "      <th>gold_standard</th>\n",
       "      <th>expert</th>\n",
       "      <th>...</th>\n",
       "      <th>time_before_submission</th>\n",
       "      <th>time_before_submission_seconds</th>\n",
       "      <th>solvability</th>\n",
       "      <th>appropriateness</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>operations</th>\n",
       "      <th>notes</th>\n",
       "      <th>topic</th>\n",
       "      <th>question_number</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>526544790</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>dfc5b8148f8ee749e72e</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-03 17:59:21 UTC</td>\n",
       "      <td>True</td>\n",
       "      <td>owner</td>\n",
       "      <td>...</td>\n",
       "      <td>0 days 00:01:19.947000</td>\n",
       "      <td>79.947</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes: This question is understandable, appropri...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>[Subtraction, Addition]</td>\n",
       "      <td></td>\n",
       "      <td>Yes</td>\n",
       "      <td>llama_q1053</td>\n",
       "      <td>llama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>526686902</td>\n",
       "      <td>not-logged-in-8d8a50a5ab4ff2a4c782</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8d8a50a5ab4ff2a4c782</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:46:57 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0 days 00:02:50.720000</td>\n",
       "      <td>170.720</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes: This question is understandable, appropri...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>[Subtraction, Addition]</td>\n",
       "      <td>The question is too simplistic</td>\n",
       "      <td>Yes</td>\n",
       "      <td>mammoth_q115</td>\n",
       "      <td>mammoth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>526687176</td>\n",
       "      <td>not-logged-in-329ddf4b8909579aafd1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>329ddf4b8909579aafd1</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:49:34 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0 days 00:02:25.379000</td>\n",
       "      <td>145.379</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No: This question contains distracting typos o...</td>\n",
       "      <td>No</td>\n",
       "      <td>[Division]</td>\n",
       "      <td>There is a contradiction in the question</td>\n",
       "      <td>Yes</td>\n",
       "      <td>llema_q801</td>\n",
       "      <td>llema</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>526687284</td>\n",
       "      <td>not-logged-in-ab81d6eefed8fef91f6b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ab81d6eefed8fef91f6b</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:50:58 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0 days 00:01:15.496000</td>\n",
       "      <td>75.496</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes: This question is understandable, appropri...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>[Multiplication]</td>\n",
       "      <td></td>\n",
       "      <td>Yes</td>\n",
       "      <td>llama_q1363</td>\n",
       "      <td>llama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>526687346</td>\n",
       "      <td>not-logged-in-02c614ded7364b0426f2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02c614ded7364b0426f2</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:51:38 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0 days 00:00:42.620000</td>\n",
       "      <td>42.620</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes: This question is understandable, appropri...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>[Subtraction, Addition]</td>\n",
       "      <td></td>\n",
       "      <td>Yes</td>\n",
       "      <td>mammoth_q75</td>\n",
       "      <td>mammoth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2485</th>\n",
       "      <td>569847408</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>2a0244cc976d3aca5b3f</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:03 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0 days 00:00:48.090000</td>\n",
       "      <td>48.090</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes: This question is understandable, appropri...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>[Addition]</td>\n",
       "      <td></td>\n",
       "      <td>Yes</td>\n",
       "      <td>egsm_only_q577</td>\n",
       "      <td>egsm_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2486</th>\n",
       "      <td>569847516</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:24 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0 days 00:00:19.165000</td>\n",
       "      <td>19.165</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No: This question is strange, confusing, conta...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>[Addition, Subtraction]</td>\n",
       "      <td></td>\n",
       "      <td>Yes</td>\n",
       "      <td>egsm_only_q434</td>\n",
       "      <td>egsm_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2487</th>\n",
       "      <td>569847621</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:42 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0 days 00:00:17.982000</td>\n",
       "      <td>17.982</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes: This question is understandable, appropri...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>[Addition, Subtraction]</td>\n",
       "      <td></td>\n",
       "      <td>Yes</td>\n",
       "      <td>egsm_only_q177</td>\n",
       "      <td>egsm_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2488</th>\n",
       "      <td>569847743</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:01 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0 days 00:00:17.620000</td>\n",
       "      <td>17.620</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No: This question is strange, confusing, conta...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>[Subtraction, Multiplication]</td>\n",
       "      <td></td>\n",
       "      <td>Yes</td>\n",
       "      <td>egsm_only_q253</td>\n",
       "      <td>egsm_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2489</th>\n",
       "      <td>569848012</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:43 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0 days 00:00:40.574000</td>\n",
       "      <td>40.574</td>\n",
       "      <td>No</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td></td>\n",
       "      <td>Yes</td>\n",
       "      <td>egsm_only_q325</td>\n",
       "      <td>egsm_only</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2490 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      classification_id                           user_name    user_id  \\\n",
       "0             526544790                         bryanchrist  2631956.0   \n",
       "1             526686902  not-logged-in-8d8a50a5ab4ff2a4c782        NaN   \n",
       "2             526687176  not-logged-in-329ddf4b8909579aafd1        NaN   \n",
       "3             526687284  not-logged-in-ab81d6eefed8fef91f6b        NaN   \n",
       "4             526687346  not-logged-in-02c614ded7364b0426f2        NaN   \n",
       "...                 ...                                 ...        ...   \n",
       "2485          569847408                         bryanchrist  2631956.0   \n",
       "2486          569847516                         bryanchrist  2631956.0   \n",
       "2487          569847621                         bryanchrist  2631956.0   \n",
       "2488          569847743                         bryanchrist  2631956.0   \n",
       "2489          569848012                         bryanchrist  2631956.0   \n",
       "\n",
       "                   user_ip  workflow_id       workflow_name  workflow_version  \\\n",
       "0     dfc5b8148f8ee749e72e        24740  Classify Questions           139.206   \n",
       "1     8d8a50a5ab4ff2a4c782        24740  Classify Questions           139.206   \n",
       "2     329ddf4b8909579aafd1        24740  Classify Questions           139.206   \n",
       "3     ab81d6eefed8fef91f6b        24740  Classify Questions           139.206   \n",
       "4     02c614ded7364b0426f2        24740  Classify Questions           139.206   \n",
       "...                    ...          ...                 ...               ...   \n",
       "2485  2a0244cc976d3aca5b3f        24740  Classify Questions           142.214   \n",
       "2486  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "2487  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "2488  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "2489  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "\n",
       "                   created_at gold_standard expert  ...  \\\n",
       "0     2023-12-03 17:59:21 UTC          True  owner  ...   \n",
       "1     2023-12-04 12:46:57 UTC           NaN    NaN  ...   \n",
       "2     2023-12-04 12:49:34 UTC           NaN    NaN  ...   \n",
       "3     2023-12-04 12:50:58 UTC           NaN    NaN  ...   \n",
       "4     2023-12-04 12:51:38 UTC           NaN    NaN  ...   \n",
       "...                       ...           ...    ...  ...   \n",
       "2485  2024-06-27 18:26:03 UTC           NaN    NaN  ...   \n",
       "2486  2024-06-27 18:26:24 UTC           NaN    NaN  ...   \n",
       "2487  2024-06-27 18:26:42 UTC           NaN    NaN  ...   \n",
       "2488  2024-06-27 18:27:01 UTC           NaN    NaN  ...   \n",
       "2489  2024-06-27 18:27:43 UTC           NaN    NaN  ...   \n",
       "\n",
       "     time_before_submission time_before_submission_seconds solvability  \\\n",
       "0    0 days 00:01:19.947000                         79.947         Yes   \n",
       "1    0 days 00:02:50.720000                        170.720         Yes   \n",
       "2    0 days 00:02:25.379000                        145.379         Yes   \n",
       "3    0 days 00:01:15.496000                         75.496         Yes   \n",
       "4    0 days 00:00:42.620000                         42.620         Yes   \n",
       "...                     ...                            ...         ...   \n",
       "2485 0 days 00:00:48.090000                         48.090         Yes   \n",
       "2486 0 days 00:00:19.165000                         19.165         Yes   \n",
       "2487 0 days 00:00:17.982000                         17.982         Yes   \n",
       "2488 0 days 00:00:17.620000                         17.620         Yes   \n",
       "2489 0 days 00:00:40.574000                         40.574          No   \n",
       "\n",
       "                                        appropriateness accuracy  \\\n",
       "0     Yes: This question is understandable, appropri...      Yes   \n",
       "1     Yes: This question is understandable, appropri...      Yes   \n",
       "2     No: This question contains distracting typos o...       No   \n",
       "3     Yes: This question is understandable, appropri...      Yes   \n",
       "4     Yes: This question is understandable, appropri...      Yes   \n",
       "...                                                 ...      ...   \n",
       "2485  Yes: This question is understandable, appropri...      Yes   \n",
       "2486  No: This question is strange, confusing, conta...      Yes   \n",
       "2487  Yes: This question is understandable, appropri...      Yes   \n",
       "2488  No: This question is strange, confusing, conta...      Yes   \n",
       "2489                                                 NA       NA   \n",
       "\n",
       "                         operations                                     notes  \\\n",
       "0           [Subtraction, Addition]                                             \n",
       "1           [Subtraction, Addition]            The question is too simplistic   \n",
       "2                        [Division]  There is a contradiction in the question   \n",
       "3                  [Multiplication]                                             \n",
       "4           [Subtraction, Addition]                                             \n",
       "...                             ...                                       ...   \n",
       "2485                     [Addition]                                             \n",
       "2486        [Addition, Subtraction]                                             \n",
       "2487        [Addition, Subtraction]                                             \n",
       "2488  [Subtraction, Multiplication]                                             \n",
       "2489                             NA                                             \n",
       "\n",
       "      topic question_number      model  \n",
       "0       Yes     llama_q1053      llama  \n",
       "1       Yes    mammoth_q115    mammoth  \n",
       "2       Yes      llema_q801      llema  \n",
       "3       Yes     llama_q1363      llama  \n",
       "4       Yes     mammoth_q75    mammoth  \n",
       "...     ...             ...        ...  \n",
       "2485    Yes  egsm_only_q577  egsm_only  \n",
       "2486    Yes  egsm_only_q434  egsm_only  \n",
       "2487    Yes  egsm_only_q177  egsm_only  \n",
       "2488    Yes  egsm_only_q253  egsm_only  \n",
       "2489    Yes  egsm_only_q325  egsm_only  \n",
       "\n",
       "[2490 rows x 26 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ec594f5-451d-484c-9e36-153bec95cfbc",
   "metadata": {
    "tags": []
   },
   "source": [
    "question_numbers = []\n",
    "model = []\n",
    "for i in range(0, len(df)):\n",
    "    my_json = json.loads(df.iloc[i]['subject_data'])\n",
    "    subject_id = df.iloc[i]['subject_ids']\n",
    "    question_number = my_json[f'{subject_id}']['Filename']\n",
    "    question_number = question_number.split(\".txt\")[0]\n",
    "    question_numbers.append(question_number)\n",
    "    if 'llema' in question_number:\n",
    "        model.append('llema')\n",
    "    if 'llama' in question_number:\n",
    "        model.append('llama')    \n",
    "    if 'mammoth' in question_number:\n",
    "        model.append('mammoth')\n",
    "    if 'mathwell' in question_number:\n",
    "        model.append('mathwell')\n",
    "    else:\n",
    "        df.drop(i)\n",
    "df['question_number']=question_numbers\n",
    "df['model']=model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d7de705-768d-47aa-99c5-6edcbe287471",
   "metadata": {},
   "source": [
    "## Explore Appropriateness Labels by Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de674092-98b2-46c5-9535-447a266ea720",
   "metadata": {},
   "source": [
    "### Overall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "950c5c77-f858-4a37-b26d-e9926f69ad5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "llama = df[df['model']=='llama']\n",
    "llema = df[df['model']=='llema']\n",
    "mathwell = df[df['model']=='mathwell']\n",
    "mammoth = df[df['model']=='mammoth']\n",
    "gpt35 = df[df['model']=='gpt35']\n",
    "gpt4 = df[df['model']=='gpt4']\n",
    "llama_gsm = df[df['model']=='llama_gsm']\n",
    "egsm_only = df[df['model']=='egsm_only']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f629c4dd-7183-417d-a6b8-9504f81b5ece",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      294\n",
       "NA                                                                                                            48\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     44\n",
       "No: The question is too hard, even for a middle school student                                                 3\n",
       "No: This question contains inappropriate material                                                              3\n",
       "No: This question contains distracting typos or grammatical errors                                             1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llama['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "890fef2f-0f8b-4c65-a2a9-4c4031e205f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "NA                                                                                                           172\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      121\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     65\n",
       "No: This question contains distracting typos or grammatical errors                                            19\n",
       "No: This question contains inappropriate material                                                              5\n",
       "No: The question is too hard, even for a middle school student                                                 1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llema['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5276c07d-9ddc-49d8-9999-e9a51ace3d70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      262\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     69\n",
       "NA                                                                                                            41\n",
       "No: This question contains distracting typos or grammatical errors                                             6\n",
       "No: This question contains inappropriate material                                                              4\n",
       "No: The question is too hard, even for a middle school student                                                 3\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mammoth['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cc65d0ec-2257-4022-b019-b8571b772fbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      326\n",
       "NA                                                                                                            41\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     33\n",
       "No: This question contains inappropriate material                                                              4\n",
       "No: The question is too hard, even for a middle school student                                                 1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mathwell['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ca7cac07-0914-4ca4-9da7-f5414a7ecb10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      207\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     42\n",
       "NA                                                                                                            33\n",
       "No: The question is too hard, even for a middle school student                                                11\n",
       "No: This question contains inappropriate material                                                              7\n",
       "No: This question contains distracting typos or grammatical errors                                             1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt35['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7ba8e882-b520-4c72-8fbe-f645a7c09e00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      219\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     24\n",
       "NA                                                                                                            15\n",
       "No: This question contains distracting typos or grammatical errors                                             6\n",
       "No: This question contains inappropriate material                                                              5\n",
       "No: The question is too hard, even for a middle school student                                                 4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt4['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53b615e3-7b75-4d80-bd58-1ba40cd34c5a",
   "metadata": {},
   "source": [
    "### Final Annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "562d505f-2b3a-43fb-973e-c4a96667476d",
   "metadata": {},
   "outputs": [],
   "source": [
    "final = df[df['user_name']==\"bryanchrist\"]\n",
    "llama_final = final[final['model']=='llama']\n",
    "llema_final = final[final['model']=='llema']\n",
    "mathwell_final = final[final['model']=='mathwell']\n",
    "mammoth_final = final[final['model']=='mammoth']\n",
    "gpt35_final = final[final['model']=='gpt35']\n",
    "gpt4_final = final[final['model']=='gpt4']\n",
    "llama_gsm_final = final[final['model']=='llama_gsm']\n",
    "egsm_only_final = final[final['model']=='egsm_only']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3df2241d-a9af-4c63-8c8a-c9245064bda1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6511627906976745 0.046511627906976744 0.046511627906976744 0.023255813953488372 0.23255813953488372 1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_strange = (llama_final['appropriateness'].value_counts()[2]-10)/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_strange = (llama_final['appropriateness'].value_counts()[2]-10)/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_strange = (llama_final['appropriateness'].value_counts()[2]-10)/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_strange = (llama_final['appropriateness'].value_counts()[2]-10)/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_strange = (llama_final['appropriateness'].value_counts()[2]-10)/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_hard = llama_final['appropriateness'].value_counts()[3]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_hard = llama_final['appropriateness'].value_counts()[3]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_hard = llama_final['appropriateness'].value_counts()[3]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_hard = llama_final['appropriateness'].value_counts()[3]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_hard = llama_final['appropriateness'].value_counts()[3]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_inappropriate = llama_final['appropriateness'].value_counts()[4]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_inappropriate = llama_final['appropriateness'].value_counts()[4]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_inappropriate = llama_final['appropriateness'].value_counts()[4]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_inappropriate = llama_final['appropriateness'].value_counts()[4]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_inappropriate = llama_final['appropriateness'].value_counts()[4]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_typo = llama_final['appropriateness'].value_counts()[5]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_typo = llama_final['appropriateness'].value_counts()[5]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_typo = llama_final['appropriateness'].value_counts()[5]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_typo = llama_final['appropriateness'].value_counts()[5]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_typo = llama_final['appropriateness'].value_counts()[5]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_no_ops = 10/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_no_ops = 10/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_no_ops = 10/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3116232776.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_no_ops = 10/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      214\n",
       "NA                                                                                                            45\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     38\n",
       "No: The question is too hard, even for a middle school student                                                 2\n",
       "No: This question contains inappropriate material                                                              2\n",
       "No: This question contains distracting typos or grammatical errors                                             1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llama_strange = (llama_final['appropriateness'].value_counts()[2]-10)/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
    "llama_hard = llama_final['appropriateness'].value_counts()[3]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
    "llama_inappropriate = llama_final['appropriateness'].value_counts()[4]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
    "llama_typo = llama_final['appropriateness'].value_counts()[5]/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
    "llama_no_ops = 10/(llama_final['appropriateness'].value_counts()[2]+llama_final['appropriateness'].value_counts()[3]+llama_final['appropriateness'].value_counts()[4]+llama_final['appropriateness'].value_counts()[5])\n",
    "print(llama_strange, llama_hard, llama_inappropriate, llama_typo, llama_no_ops, llama_strange+llama_hard+llama_inappropriate+llama_typo+llama_no_ops)\n",
    "llama_final['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d83a5344-7302-4a20-a944-0a42dae8024c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.410958904109589 0.2465753424657534 0.0684931506849315 0.0136986301369863 0.2602739726027397 1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_strange = (llema_final['appropriateness'].value_counts()[2]-19)/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_strange = (llema_final['appropriateness'].value_counts()[2]-19)/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_strange = (llema_final['appropriateness'].value_counts()[2]-19)/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_strange = (llema_final['appropriateness'].value_counts()[2]-19)/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_strange = (llema_final['appropriateness'].value_counts()[2]-19)/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_hard = llema_final['appropriateness'].value_counts()[3]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_hard = llema_final['appropriateness'].value_counts()[3]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_hard = llema_final['appropriateness'].value_counts()[3]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_hard = llema_final['appropriateness'].value_counts()[3]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_hard = llema_final['appropriateness'].value_counts()[3]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_inappropriate = llema_final['appropriateness'].value_counts()[4]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_inappropriate = llema_final['appropriateness'].value_counts()[4]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_inappropriate = llema_final['appropriateness'].value_counts()[4]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_inappropriate = llema_final['appropriateness'].value_counts()[4]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_inappropriate = llema_final['appropriateness'].value_counts()[4]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_typo = llema_final['appropriateness'].value_counts()[5]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_typo = llema_final['appropriateness'].value_counts()[5]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_typo = llema_final['appropriateness'].value_counts()[5]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_typo = llema_final['appropriateness'].value_counts()[5]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_typo = llema_final['appropriateness'].value_counts()[5]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_no_ops = 19/ (llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_no_ops = 19/ (llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_no_ops = 19/ (llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/188116.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llema_no_ops = 19/ (llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "NA                                                                                                           139\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it       66\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     49\n",
       "No: This question contains distracting typos or grammatical errors                                            18\n",
       "No: This question contains inappropriate material                                                              5\n",
       "No: The question is too hard, even for a middle school student                                                 1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llema_strange = (llema_final['appropriateness'].value_counts()[2]-19)/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
    "llema_hard = llema_final['appropriateness'].value_counts()[3]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
    "llema_inappropriate = llema_final['appropriateness'].value_counts()[4]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
    "llema_typo = llema_final['appropriateness'].value_counts()[5]/(llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
    "llema_no_ops = 19/ (llema_final['appropriateness'].value_counts()[2]+llema_final['appropriateness'].value_counts()[3]+llema_final['appropriateness'].value_counts()[4]+llema_final['appropriateness'].value_counts()[5])\n",
    "print(llema_strange, llema_hard, llema_inappropriate, llema_typo, llema_no_ops, llema_strange+llema_hard+llema_inappropriate+llema_typo+llema_no_ops)\n",
    "llema_final['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e6c9dfb4-12bb-48d3-995e-24f0853cab6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7741935483870968 0.12903225806451613 0.0967741935483871 1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1118802284.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mathwell_strange = (mathwell_final['appropriateness'].value_counts()[2]-3)/(mathwell_final['appropriateness'].value_counts()[2]+mathwell_final['appropriateness'].value_counts()[3])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1118802284.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mathwell_strange = (mathwell_final['appropriateness'].value_counts()[2]-3)/(mathwell_final['appropriateness'].value_counts()[2]+mathwell_final['appropriateness'].value_counts()[3])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1118802284.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mathwell_strange = (mathwell_final['appropriateness'].value_counts()[2]-3)/(mathwell_final['appropriateness'].value_counts()[2]+mathwell_final['appropriateness'].value_counts()[3])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1118802284.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mathwell_typo = mathwell_final['appropriateness'].value_counts()[3]/(mathwell_final['appropriateness'].value_counts()[2]+mathwell_final['appropriateness'].value_counts()[3])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1118802284.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mathwell_typo = mathwell_final['appropriateness'].value_counts()[3]/(mathwell_final['appropriateness'].value_counts()[2]+mathwell_final['appropriateness'].value_counts()[3])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1118802284.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mathwell_typo = mathwell_final['appropriateness'].value_counts()[3]/(mathwell_final['appropriateness'].value_counts()[2]+mathwell_final['appropriateness'].value_counts()[3])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1118802284.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mathwell_no_ops = 3/(mathwell_final['appropriateness'].value_counts()[2]+mathwell_final['appropriateness'].value_counts()[3])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1118802284.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mathwell_no_ops = 3/(mathwell_final['appropriateness'].value_counts()[2]+mathwell_final['appropriateness'].value_counts()[3])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      237\n",
       "NA                                                                                                            36\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     27\n",
       "No: This question contains inappropriate material                                                              4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mathwell_strange = (mathwell_final['appropriateness'].value_counts()[2]-3)/(mathwell_final['appropriateness'].value_counts()[2]+mathwell_final['appropriateness'].value_counts()[3])\n",
    "mathwell_typo = mathwell_final['appropriateness'].value_counts()[3]/(mathwell_final['appropriateness'].value_counts()[2]+mathwell_final['appropriateness'].value_counts()[3])\n",
    "mathwell_no_ops = 3/(mathwell_final['appropriateness'].value_counts()[2]+mathwell_final['appropriateness'].value_counts()[3])\n",
    "print(mathwell_strange, mathwell_typo, mathwell_no_ops, mathwell_strange+mathwell_typo+mathwell_no_ops)\n",
    "mathwell_final['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8276f3da-c8a1-4069-8251-76029dbc79f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7792207792207793 0.07792207792207792 0.05194805194805195 0.025974025974025976 0.06493506493506493 1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_strange = (mammoth_final['appropriateness'].value_counts()[1]-5)/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_strange = (mammoth_final['appropriateness'].value_counts()[1]-5)/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_strange = (mammoth_final['appropriateness'].value_counts()[1]-5)/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_strange = (mammoth_final['appropriateness'].value_counts()[1]-5)/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_strange = (mammoth_final['appropriateness'].value_counts()[1]-5)/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_hard = mammoth_final['appropriateness'].value_counts()[3]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_hard = mammoth_final['appropriateness'].value_counts()[3]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_hard = mammoth_final['appropriateness'].value_counts()[3]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_hard = mammoth_final['appropriateness'].value_counts()[3]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_hard = mammoth_final['appropriateness'].value_counts()[3]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_inappropriate = mammoth_final['appropriateness'].value_counts()[4]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_inappropriate = mammoth_final['appropriateness'].value_counts()[4]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_inappropriate = mammoth_final['appropriateness'].value_counts()[4]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_inappropriate = mammoth_final['appropriateness'].value_counts()[4]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_inappropriate = mammoth_final['appropriateness'].value_counts()[4]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_typo = mammoth_final['appropriateness'].value_counts()[5]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_typo = mammoth_final['appropriateness'].value_counts()[5]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_typo = mammoth_final['appropriateness'].value_counts()[5]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_typo = mammoth_final['appropriateness'].value_counts()[5]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_typo = mammoth_final['appropriateness'].value_counts()[5]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_no_ops = 5/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_no_ops = 5/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_no_ops = 5/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3910302365.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  mammoth_no_ops = 5/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      160\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     65\n",
       "NA                                                                                                            34\n",
       "No: This question contains distracting typos or grammatical errors                                             6\n",
       "No: This question contains inappropriate material                                                              4\n",
       "No: The question is too hard, even for a middle school student                                                 2\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mammoth_strange = (mammoth_final['appropriateness'].value_counts()[1]-5)/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
    "mammoth_hard = mammoth_final['appropriateness'].value_counts()[3]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
    "mammoth_inappropriate = mammoth_final['appropriateness'].value_counts()[4]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
    "mammoth_typo = mammoth_final['appropriateness'].value_counts()[5]/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
    "mammoth_no_ops = 5/(mammoth_final['appropriateness'].value_counts()[1]+mammoth_final['appropriateness'].value_counts()[3]+mammoth_final['appropriateness'].value_counts()[4]+mammoth_final['appropriateness'].value_counts()[5])\n",
    "print(mammoth_strange, mammoth_hard, mammoth_inappropriate, mammoth_typo, mammoth_no_ops, mammoth_strange+mammoth_hard+mammoth_inappropriate+mammoth_typo+mammoth_no_ops)\n",
    "mammoth_final['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "2e007ae3-014e-48a9-8771-7b2026152448",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6071428571428571 0.17857142857142858 0.125 0.017857142857142856 0.07142857142857142 1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_strange = (gpt35_final['appropriateness'].value_counts()[1]-4)/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_strange = (gpt35_final['appropriateness'].value_counts()[1]-4)/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_strange = (gpt35_final['appropriateness'].value_counts()[1]-4)/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_strange = (gpt35_final['appropriateness'].value_counts()[1]-4)/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_strange = (gpt35_final['appropriateness'].value_counts()[1]-4)/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_hard = gpt35_final['appropriateness'].value_counts()[3]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_hard = gpt35_final['appropriateness'].value_counts()[3]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_hard = gpt35_final['appropriateness'].value_counts()[3]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_hard = gpt35_final['appropriateness'].value_counts()[3]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_hard = gpt35_final['appropriateness'].value_counts()[3]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_inappropriate = gpt35_final['appropriateness'].value_counts()[4]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_inappropriate = gpt35_final['appropriateness'].value_counts()[4]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_inappropriate = gpt35_final['appropriateness'].value_counts()[4]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_inappropriate = gpt35_final['appropriateness'].value_counts()[4]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_inappropriate = gpt35_final['appropriateness'].value_counts()[4]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_typo = gpt35_final['appropriateness'].value_counts()[5]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_typo = gpt35_final['appropriateness'].value_counts()[5]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_typo = gpt35_final['appropriateness'].value_counts()[5]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_typo = gpt35_final['appropriateness'].value_counts()[5]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_typo = gpt35_final['appropriateness'].value_counts()[5]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_no_ops = 4/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_no_ops = 4/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_no_ops = 4/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/159065396.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt35_no_ops = 4/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      165\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     38\n",
       "NA                                                                                                            29\n",
       "No: The question is too hard, even for a middle school student                                                10\n",
       "No: This question contains inappropriate material                                                              7\n",
       "No: This question contains distracting typos or grammatical errors                                             1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt35_strange = (gpt35_final['appropriateness'].value_counts()[1]-4)/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
    "gpt35_hard = gpt35_final['appropriateness'].value_counts()[3]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
    "gpt35_inappropriate = gpt35_final['appropriateness'].value_counts()[4]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
    "gpt35_typo = gpt35_final['appropriateness'].value_counts()[5]/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
    "gpt35_no_ops = 4/(gpt35_final['appropriateness'].value_counts()[1]+gpt35_final['appropriateness'].value_counts()[3]+gpt35_final['appropriateness'].value_counts()[4]+gpt35_final['appropriateness'].value_counts()[5])\n",
    "print(gpt35_strange, gpt35_hard, gpt35_inappropriate, gpt35_typo, gpt35_no_ops, gpt35_strange+gpt35_hard+gpt35_inappropriate+gpt35_typo+gpt35_no_ops)\n",
    "gpt35_final['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "78452919-62da-462b-b55f-5428356bf854",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5526315789473685 0.13157894736842105 0.13157894736842105 0.10526315789473684 0.07894736842105263 1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_strange = (gpt4_final['appropriateness'].value_counts()[1]-3)/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_strange = (gpt4_final['appropriateness'].value_counts()[1]-3)/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_strange = (gpt4_final['appropriateness'].value_counts()[1]-3)/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_strange = (gpt4_final['appropriateness'].value_counts()[1]-3)/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_strange = (gpt4_final['appropriateness'].value_counts()[1]-3)/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_hard = gpt4_final['appropriateness'].value_counts()[3]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_hard = gpt4_final['appropriateness'].value_counts()[3]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_hard = gpt4_final['appropriateness'].value_counts()[3]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_hard = gpt4_final['appropriateness'].value_counts()[3]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_hard = gpt4_final['appropriateness'].value_counts()[3]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_inappropriate = gpt4_final['appropriateness'].value_counts()[4]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_inappropriate = gpt4_final['appropriateness'].value_counts()[4]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_inappropriate = gpt4_final['appropriateness'].value_counts()[4]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_inappropriate = gpt4_final['appropriateness'].value_counts()[4]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_inappropriate = gpt4_final['appropriateness'].value_counts()[4]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_typo = gpt4_final['appropriateness'].value_counts()[5]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_typo = gpt4_final['appropriateness'].value_counts()[5]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_typo = gpt4_final['appropriateness'].value_counts()[5]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_typo = gpt4_final['appropriateness'].value_counts()[5]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_typo = gpt4_final['appropriateness'].value_counts()[5]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_no_ops = 3/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_no_ops = 3/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_no_ops = 3/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/1723831084.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  gpt4_no_ops = 3/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      198\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     24\n",
       "NA                                                                                                            14\n",
       "No: This question contains inappropriate material                                                              5\n",
       "No: This question contains distracting typos or grammatical errors                                             5\n",
       "No: The question is too hard, even for a middle school student                                                 4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt4_strange = (gpt4_final['appropriateness'].value_counts()[1]-3)/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
    "gpt4_hard = gpt4_final['appropriateness'].value_counts()[3]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
    "gpt4_inappropriate = gpt4_final['appropriateness'].value_counts()[4]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
    "gpt4_typo = gpt4_final['appropriateness'].value_counts()[5]/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
    "gpt4_no_ops = 3/(gpt4_final['appropriateness'].value_counts()[1]+gpt4_final['appropriateness'].value_counts()[3]+gpt4_final['appropriateness'].value_counts()[4]+gpt4_final['appropriateness'].value_counts()[5])\n",
    "print(gpt4_strange, gpt4_hard, gpt4_inappropriate, gpt4_typo, gpt4_no_ops, gpt4_strange+gpt4_hard+gpt4_inappropriate+gpt4_typo+gpt4_no_ops)\n",
    "gpt4_final['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "aefe9434-e7e8-4101-99f4-63d04414ddba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5882352941176471 0.11764705882352941 0.08823529411764706 0.029411764705882353 0.17647058823529413 1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_strange = (llama_gsm_final['appropriateness'].value_counts()[2]-6)/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_strange = (llama_gsm_final['appropriateness'].value_counts()[2]-6)/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_strange = (llama_gsm_final['appropriateness'].value_counts()[2]-6)/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_strange = (llama_gsm_final['appropriateness'].value_counts()[2]-6)/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_strange = (llama_gsm_final['appropriateness'].value_counts()[2]-6)/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_hard = llama_gsm_final['appropriateness'].value_counts()[3]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_hard = llama_gsm_final['appropriateness'].value_counts()[3]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_hard = llama_gsm_final['appropriateness'].value_counts()[3]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_hard = llama_gsm_final['appropriateness'].value_counts()[3]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_hard = llama_gsm_final['appropriateness'].value_counts()[3]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_inappropriate = llama_gsm_final['appropriateness'].value_counts()[4]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_inappropriate = llama_gsm_final['appropriateness'].value_counts()[4]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_inappropriate = llama_gsm_final['appropriateness'].value_counts()[4]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_inappropriate = llama_gsm_final['appropriateness'].value_counts()[4]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_inappropriate = llama_gsm_final['appropriateness'].value_counts()[4]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_typo = llama_gsm_final['appropriateness'].value_counts()[5]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_typo = llama_gsm_final['appropriateness'].value_counts()[5]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_typo = llama_gsm_final['appropriateness'].value_counts()[5]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_typo = llama_gsm_final['appropriateness'].value_counts()[5]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_typo = llama_gsm_final['appropriateness'].value_counts()[5]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_no_ops = 6/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_no_ops = 6/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_no_ops = 6/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4162829102.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  llama_gsm_no_ops = 6/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      40\n",
       "NA                                                                                                           26\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality    26\n",
       "No: The question is too hard, even for a middle school student                                                4\n",
       "No: This question contains inappropriate material                                                             3\n",
       "No: This question contains distracting typos or grammatical errors                                            1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llama_gsm_strange = (llama_gsm_final['appropriateness'].value_counts()[2]-6)/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
    "llama_gsm_hard = llama_gsm_final['appropriateness'].value_counts()[3]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
    "llama_gsm_inappropriate = llama_gsm_final['appropriateness'].value_counts()[4]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
    "llama_gsm_typo = llama_gsm_final['appropriateness'].value_counts()[5]/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
    "llama_gsm_no_ops = 6/(llama_gsm_final['appropriateness'].value_counts()[2]+llama_gsm_final['appropriateness'].value_counts()[3]+llama_gsm_final['appropriateness'].value_counts()[4]+llama_gsm_final['appropriateness'].value_counts()[5])\n",
    "print(llama_gsm_strange, llama_gsm_hard, llama_gsm_inappropriate, llama_gsm_typo, llama_gsm_no_ops, llama_gsm_strange+llama_gsm_hard+llama_gsm_inappropriate+llama_gsm_typo+llama_gsm_no_ops)\n",
    "llama_gsm_final['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "9816b7da-07d6-4e11-877e-a9da375be605",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6 0.13333333333333333 0.06666666666666667 0.022222222222222223 0.17777777777777778 1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_strange = (egsm_only_final['appropriateness'].value_counts()[2]-8)/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_strange = (egsm_only_final['appropriateness'].value_counts()[2]-8)/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_strange = (egsm_only_final['appropriateness'].value_counts()[2]-8)/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_strange = (egsm_only_final['appropriateness'].value_counts()[2]-8)/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_strange = (egsm_only_final['appropriateness'].value_counts()[2]-8)/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_hard = egsm_only_final['appropriateness'].value_counts()[3]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_hard = egsm_only_final['appropriateness'].value_counts()[3]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_hard = egsm_only_final['appropriateness'].value_counts()[3]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_hard = egsm_only_final['appropriateness'].value_counts()[3]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_hard = egsm_only_final['appropriateness'].value_counts()[3]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_inappropriate = egsm_only_final['appropriateness'].value_counts()[4]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_inappropriate = egsm_only_final['appropriateness'].value_counts()[4]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_inappropriate = egsm_only_final['appropriateness'].value_counts()[4]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_inappropriate = egsm_only_final['appropriateness'].value_counts()[4]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_inappropriate = egsm_only_final['appropriateness'].value_counts()[4]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_typo = egsm_only_final['appropriateness'].value_counts()[5]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_typo = egsm_only_final['appropriateness'].value_counts()[5]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_typo = egsm_only_final['appropriateness'].value_counts()[5]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_typo = egsm_only_final['appropriateness'].value_counts()[5]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:4: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_typo = egsm_only_final['appropriateness'].value_counts()[5]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_no_ops = 8/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_no_ops = 8/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_no_ops = 8/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/842494450.py:5: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  egsm_only_no_ops = 8/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "appropriateness\n",
       "Yes: This question is understandable, appropriate and a middle school student or younger could solve it      159\n",
       "No: This question is strange, confusing, contains conflicting information, and/or is not based in reality     46\n",
       "NA                                                                                                            35\n",
       "No: This question contains distracting typos or grammatical errors                                             6\n",
       "No: This question contains inappropriate material                                                              3\n",
       "No: The question is too hard, even for a middle school student                                                 1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "egsm_only_strange = (egsm_only_final['appropriateness'].value_counts()[2]-8)/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
    "egsm_only_hard = egsm_only_final['appropriateness'].value_counts()[3]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
    "egsm_only_inappropriate = egsm_only_final['appropriateness'].value_counts()[4]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
    "egsm_only_typo = egsm_only_final['appropriateness'].value_counts()[5]/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
    "egsm_only_no_ops = 8/(egsm_only_final['appropriateness'].value_counts()[2]+egsm_only_final['appropriateness'].value_counts()[3]+egsm_only_final['appropriateness'].value_counts()[4]+egsm_only_final['appropriateness'].value_counts()[5])\n",
    "print(egsm_only_strange, egsm_only_hard, egsm_only_inappropriate, egsm_only_typo, egsm_only_no_ops, egsm_only_strange+egsm_only_hard+egsm_only_inappropriate+egsm_only_typo+egsm_only_no_ops)\n",
    "egsm_only_final['appropriateness'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9132c753-3417-48c1-9f81-2c17f6c1d892",
   "metadata": {},
   "source": [
    "## Relabel/Recode Variables and Calculate MaC/Operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2283d96a-529d-4422-ac44-924d7783a342",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4060560311.py:3: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n",
      "  df['solvability'] = df['solvability'].replace(replace_map)\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4060560311.py:4: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n",
      "  df['accuracy'] = df['accuracy'].replace(replace_map)\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4060560311.py:5: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n",
      "  df['topic']=df['topic'].replace(replace_map)\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4060560311.py:18: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n",
      "  df['appropriateness'] = df['appropriateness'].replace(replace_map)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>classification_id</th>\n",
       "      <th>user_name</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_ip</th>\n",
       "      <th>workflow_id</th>\n",
       "      <th>workflow_name</th>\n",
       "      <th>workflow_version</th>\n",
       "      <th>created_at</th>\n",
       "      <th>gold_standard</th>\n",
       "      <th>expert</th>\n",
       "      <th>...</th>\n",
       "      <th>model</th>\n",
       "      <th>addition</th>\n",
       "      <th>subtraction</th>\n",
       "      <th>multiplication</th>\n",
       "      <th>division</th>\n",
       "      <th>fractions</th>\n",
       "      <th>decimals</th>\n",
       "      <th>no_ops</th>\n",
       "      <th>total_ops</th>\n",
       "      <th>good</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>526544790</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>dfc5b8148f8ee749e72e</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-03 17:59:21 UTC</td>\n",
       "      <td>True</td>\n",
       "      <td>owner</td>\n",
       "      <td>...</td>\n",
       "      <td>llama</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>526686902</td>\n",
       "      <td>not-logged-in-8d8a50a5ab4ff2a4c782</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8d8a50a5ab4ff2a4c782</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:46:57 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>mammoth</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>526687176</td>\n",
       "      <td>not-logged-in-329ddf4b8909579aafd1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>329ddf4b8909579aafd1</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:49:34 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>llema</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>526687284</td>\n",
       "      <td>not-logged-in-ab81d6eefed8fef91f6b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ab81d6eefed8fef91f6b</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:50:58 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>llama</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>526687346</td>\n",
       "      <td>not-logged-in-02c614ded7364b0426f2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02c614ded7364b0426f2</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:51:38 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>mammoth</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2485</th>\n",
       "      <td>569847408</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>2a0244cc976d3aca5b3f</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:03 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>egsm_only</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2486</th>\n",
       "      <td>569847516</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:24 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>egsm_only</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2487</th>\n",
       "      <td>569847621</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:42 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>egsm_only</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2488</th>\n",
       "      <td>569847743</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:01 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>egsm_only</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2489</th>\n",
       "      <td>569848012</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:43 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>egsm_only</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2490 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      classification_id                           user_name    user_id  \\\n",
       "0             526544790                         bryanchrist  2631956.0   \n",
       "1             526686902  not-logged-in-8d8a50a5ab4ff2a4c782        NaN   \n",
       "2             526687176  not-logged-in-329ddf4b8909579aafd1        NaN   \n",
       "3             526687284  not-logged-in-ab81d6eefed8fef91f6b        NaN   \n",
       "4             526687346  not-logged-in-02c614ded7364b0426f2        NaN   \n",
       "...                 ...                                 ...        ...   \n",
       "2485          569847408                         bryanchrist  2631956.0   \n",
       "2486          569847516                         bryanchrist  2631956.0   \n",
       "2487          569847621                         bryanchrist  2631956.0   \n",
       "2488          569847743                         bryanchrist  2631956.0   \n",
       "2489          569848012                         bryanchrist  2631956.0   \n",
       "\n",
       "                   user_ip  workflow_id       workflow_name  workflow_version  \\\n",
       "0     dfc5b8148f8ee749e72e        24740  Classify Questions           139.206   \n",
       "1     8d8a50a5ab4ff2a4c782        24740  Classify Questions           139.206   \n",
       "2     329ddf4b8909579aafd1        24740  Classify Questions           139.206   \n",
       "3     ab81d6eefed8fef91f6b        24740  Classify Questions           139.206   \n",
       "4     02c614ded7364b0426f2        24740  Classify Questions           139.206   \n",
       "...                    ...          ...                 ...               ...   \n",
       "2485  2a0244cc976d3aca5b3f        24740  Classify Questions           142.214   \n",
       "2486  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "2487  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "2488  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "2489  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "\n",
       "                   created_at gold_standard expert  ...      model addition  \\\n",
       "0     2023-12-03 17:59:21 UTC          True  owner  ...      llama      1.0   \n",
       "1     2023-12-04 12:46:57 UTC           NaN    NaN  ...    mammoth      1.0   \n",
       "2     2023-12-04 12:49:34 UTC           NaN    NaN  ...      llema      0.0   \n",
       "3     2023-12-04 12:50:58 UTC           NaN    NaN  ...      llama      0.0   \n",
       "4     2023-12-04 12:51:38 UTC           NaN    NaN  ...    mammoth      1.0   \n",
       "...                       ...           ...    ...  ...        ...      ...   \n",
       "2485  2024-06-27 18:26:03 UTC           NaN    NaN  ...  egsm_only      1.0   \n",
       "2486  2024-06-27 18:26:24 UTC           NaN    NaN  ...  egsm_only      1.0   \n",
       "2487  2024-06-27 18:26:42 UTC           NaN    NaN  ...  egsm_only      1.0   \n",
       "2488  2024-06-27 18:27:01 UTC           NaN    NaN  ...  egsm_only      0.0   \n",
       "2489  2024-06-27 18:27:43 UTC           NaN    NaN  ...  egsm_only      NaN   \n",
       "\n",
       "     subtraction  multiplication division fractions decimals  no_ops  \\\n",
       "0            1.0             0.0      0.0       0.0      0.0     0.0   \n",
       "1            1.0             0.0      0.0       0.0      0.0     0.0   \n",
       "2            0.0             0.0      1.0       0.0      0.0     0.0   \n",
       "3            0.0             1.0      0.0       0.0      0.0     0.0   \n",
       "4            1.0             0.0      0.0       0.0      0.0     0.0   \n",
       "...          ...             ...      ...       ...      ...     ...   \n",
       "2485         0.0             0.0      0.0       0.0      0.0     0.0   \n",
       "2486         1.0             0.0      0.0       0.0      0.0     0.0   \n",
       "2487         1.0             0.0      0.0       0.0      0.0     0.0   \n",
       "2488         1.0             1.0      0.0       0.0      0.0     0.0   \n",
       "2489         NaN             NaN      NaN       NaN      NaN     NaN   \n",
       "\n",
       "      total_ops  good  \n",
       "0           2.0     1  \n",
       "1           2.0     1  \n",
       "2           1.0     0  \n",
       "3           1.0     1  \n",
       "4           2.0     1  \n",
       "...         ...   ...  \n",
       "2485        1.0     1  \n",
       "2486        2.0     0  \n",
       "2487        2.0     1  \n",
       "2488        2.0     0  \n",
       "2489        NaN     0  \n",
       "\n",
       "[2490 rows x 35 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "replace_map = {\"Yes\": 1, \"No\": 0, \"NA\": np.nan}\n",
    "df['solvability'] = df['solvability'].replace(replace_map)\n",
    "df['accuracy'] = df['accuracy'].replace(replace_map)\n",
    "df['topic']=df['topic'].replace(replace_map)\n",
    "replace_map = {\"Yes: This question is understandable, appropriate and a middle school student or younger could solve it \": 1, \n",
    "               \"No: This question is strange, confusing, contains conflicting information, and/or is not based in reality\": 0, \n",
    "               \"No: This question contains distracting typos or grammatical errors\": 0, \n",
    "               \"No: This question contains inappropriate material\": 0, \n",
    "              \"No: The question is too hard, even for a middle school student\": 0, \n",
    "            \"No: This question is strange, confusing, contains conflicting information, and/or is not based in reality\":0,\n",
    "            \"No: This question contains inappropriate material\":0,\n",
    "                \"No: The question is too hard, even for a middle school student\":0,\n",
    "            \"No: This question contains distracting typos or grammatical errors\": 0,\n",
    "            \"No: This question is strange, contains confusing or conflicting wording, and/or is not based in reality\":0,\n",
    "                \"No: This question is strange or confusing\":0,\n",
    "              \"NA\": np.nan}\n",
    "df['appropriateness'] = df['appropriateness'].replace(replace_map)\n",
    "def addition(df):\n",
    "    if 'Addition' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def subtraction(df):\n",
    "    if 'Subtraction' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def multiplication(df):\n",
    "    if 'Multiplication' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def division(df):\n",
    "    if 'Division' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def fractions(df):\n",
    "    if 'Fractions' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def decimals(df):\n",
    "    if 'Decimals' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def none(df):\n",
    "    if 'Not Applicable: This question does not require a mathematical operation.' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def total_ops(df):\n",
    "    if df['no_ops']==1:\n",
    "        return np.nan\n",
    "    else:\n",
    "        return df['addition'] + df['subtraction'] + df['multiplication'] + df['decimals'] + df['division'] + df['fractions']\n",
    "df['addition'] = df.apply(addition, axis=1)\n",
    "df['subtraction'] = df.apply(subtraction, axis=1)\n",
    "df['multiplication'] = df.apply(multiplication, axis=1)\n",
    "df['division'] = df.apply(division, axis=1)\n",
    "df['fractions'] = df.apply(fractions, axis=1)\n",
    "df['decimals'] = df.apply(decimals, axis=1)\n",
    "df['no_ops'] = df.apply(none, axis=1)\n",
    "df['total_ops'] = df.apply(total_ops, axis=1)\n",
    "def good(df):\n",
    "    if df['solvability']==1:\n",
    "        if df['accuracy']==1:\n",
    "            if df['no_ops']==0:\n",
    "                if df['appropriateness']==1:\n",
    "                   # if df['topic']==1:\n",
    "                    return 1\n",
    "    if df['accuracy']==np.nan:\n",
    "        return 0\n",
    "    if df['no_ops']==np.nan:\n",
    "        return 0\n",
    "    if df['appropriateness']==np.nan:\n",
    "        return 0\n",
    "    else:\n",
    "        return 0\n",
    "df['good'] = df.apply(good, axis = 1)\n",
    "def good_recode(df):\n",
    "    if df['good']==np.nan:\n",
    "        return 0\n",
    "    else:\n",
    "        return df['good']\n",
    "df['good'] = df.apply(good_recode, axis=1)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a255c13b-d94b-4d21-b219-ffae5bca7e6d",
   "metadata": {},
   "source": [
    "## Check Annotator Agreement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "1943c6f0-433c-485a-9e4a-d198d8986742",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "359 60\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>classification_id</th>\n",
       "      <th>user_name</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_ip</th>\n",
       "      <th>workflow_id</th>\n",
       "      <th>workflow_name</th>\n",
       "      <th>workflow_version</th>\n",
       "      <th>created_at</th>\n",
       "      <th>gold_standard</th>\n",
       "      <th>expert</th>\n",
       "      <th>...</th>\n",
       "      <th>model</th>\n",
       "      <th>addition</th>\n",
       "      <th>subtraction</th>\n",
       "      <th>multiplication</th>\n",
       "      <th>division</th>\n",
       "      <th>fractions</th>\n",
       "      <th>decimals</th>\n",
       "      <th>no_ops</th>\n",
       "      <th>total_ops</th>\n",
       "      <th>good</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>526544790</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>dfc5b8148f8ee749e72e</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-03 17:59:21 UTC</td>\n",
       "      <td>True</td>\n",
       "      <td>owner</td>\n",
       "      <td>...</td>\n",
       "      <td>llama</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>526686902</td>\n",
       "      <td>not-logged-in-8d8a50a5ab4ff2a4c782</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8d8a50a5ab4ff2a4c782</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:46:57 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>mammoth</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>526687176</td>\n",
       "      <td>not-logged-in-329ddf4b8909579aafd1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>329ddf4b8909579aafd1</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:49:34 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>llema</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>526687284</td>\n",
       "      <td>not-logged-in-ab81d6eefed8fef91f6b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ab81d6eefed8fef91f6b</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:50:58 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>llama</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>526687346</td>\n",
       "      <td>not-logged-in-02c614ded7364b0426f2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02c614ded7364b0426f2</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>139.206</td>\n",
       "      <td>2023-12-04 12:51:38 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>mammoth</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2485</th>\n",
       "      <td>569847408</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>2a0244cc976d3aca5b3f</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:03 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>egsm_only</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2486</th>\n",
       "      <td>569847516</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:24 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>egsm_only</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2487</th>\n",
       "      <td>569847621</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:26:42 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>egsm_only</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2488</th>\n",
       "      <td>569847743</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:01 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>egsm_only</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2489</th>\n",
       "      <td>569848012</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>2631956.0</td>\n",
       "      <td>4d1e22248ff8d5ac8526</td>\n",
       "      <td>24740</td>\n",
       "      <td>Classify Questions</td>\n",
       "      <td>142.214</td>\n",
       "      <td>2024-06-27 18:27:43 UTC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>egsm_only</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2410 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      classification_id                           user_name    user_id  \\\n",
       "0             526544790                         bryanchrist  2631956.0   \n",
       "1             526686902  not-logged-in-8d8a50a5ab4ff2a4c782        NaN   \n",
       "2             526687176  not-logged-in-329ddf4b8909579aafd1        NaN   \n",
       "3             526687284  not-logged-in-ab81d6eefed8fef91f6b        NaN   \n",
       "4             526687346  not-logged-in-02c614ded7364b0426f2        NaN   \n",
       "...                 ...                                 ...        ...   \n",
       "2485          569847408                         bryanchrist  2631956.0   \n",
       "2486          569847516                         bryanchrist  2631956.0   \n",
       "2487          569847621                         bryanchrist  2631956.0   \n",
       "2488          569847743                         bryanchrist  2631956.0   \n",
       "2489          569848012                         bryanchrist  2631956.0   \n",
       "\n",
       "                   user_ip  workflow_id       workflow_name  workflow_version  \\\n",
       "0     dfc5b8148f8ee749e72e        24740  Classify Questions           139.206   \n",
       "1     8d8a50a5ab4ff2a4c782        24740  Classify Questions           139.206   \n",
       "2     329ddf4b8909579aafd1        24740  Classify Questions           139.206   \n",
       "3     ab81d6eefed8fef91f6b        24740  Classify Questions           139.206   \n",
       "4     02c614ded7364b0426f2        24740  Classify Questions           139.206   \n",
       "...                    ...          ...                 ...               ...   \n",
       "2485  2a0244cc976d3aca5b3f        24740  Classify Questions           142.214   \n",
       "2486  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "2487  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "2488  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "2489  4d1e22248ff8d5ac8526        24740  Classify Questions           142.214   \n",
       "\n",
       "                   created_at gold_standard expert  ...      model addition  \\\n",
       "0     2023-12-03 17:59:21 UTC          True  owner  ...      llama      1.0   \n",
       "1     2023-12-04 12:46:57 UTC           NaN    NaN  ...    mammoth      1.0   \n",
       "2     2023-12-04 12:49:34 UTC           NaN    NaN  ...      llema      0.0   \n",
       "3     2023-12-04 12:50:58 UTC           NaN    NaN  ...      llama      0.0   \n",
       "4     2023-12-04 12:51:38 UTC           NaN    NaN  ...    mammoth      1.0   \n",
       "...                       ...           ...    ...  ...        ...      ...   \n",
       "2485  2024-06-27 18:26:03 UTC           NaN    NaN  ...  egsm_only      1.0   \n",
       "2486  2024-06-27 18:26:24 UTC           NaN    NaN  ...  egsm_only      1.0   \n",
       "2487  2024-06-27 18:26:42 UTC           NaN    NaN  ...  egsm_only      1.0   \n",
       "2488  2024-06-27 18:27:01 UTC           NaN    NaN  ...  egsm_only      0.0   \n",
       "2489  2024-06-27 18:27:43 UTC           NaN    NaN  ...  egsm_only      NaN   \n",
       "\n",
       "     subtraction  multiplication division fractions decimals  no_ops  \\\n",
       "0            1.0             0.0      0.0       0.0      0.0     0.0   \n",
       "1            1.0             0.0      0.0       0.0      0.0     0.0   \n",
       "2            0.0             0.0      1.0       0.0      0.0     0.0   \n",
       "3            0.0             1.0      0.0       0.0      0.0     0.0   \n",
       "4            1.0             0.0      0.0       0.0      0.0     0.0   \n",
       "...          ...             ...      ...       ...      ...     ...   \n",
       "2485         0.0             0.0      0.0       0.0      0.0     0.0   \n",
       "2486         1.0             0.0      0.0       0.0      0.0     0.0   \n",
       "2487         1.0             0.0      0.0       0.0      0.0     0.0   \n",
       "2488         1.0             1.0      0.0       0.0      0.0     0.0   \n",
       "2489         NaN             NaN      NaN       NaN      NaN     NaN   \n",
       "\n",
       "      total_ops  good  \n",
       "0           2.0     1  \n",
       "1           2.0     1  \n",
       "2           1.0     0  \n",
       "3           1.0     1  \n",
       "4           2.0     1  \n",
       "...         ...   ...  \n",
       "2485        1.0     1  \n",
       "2486        2.0     0  \n",
       "2487        2.0     1  \n",
       "2488        2.0     0  \n",
       "2489        NaN     0  \n",
       "\n",
       "[2410 rows x 35 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop_duplicates(subset = ['question_number', 'user_name'])\n",
    "#df = df.query(\"user_name=='bryanchrist' or user_name=='Raceaturtle'\")\n",
    "#df = df[df['user_name']!='kidleader']\n",
    "#df = df[df['user_name']!='cate11']\n",
    "print((df['question_number'].value_counts() == 2).sum(), (df['question_number'].value_counts() == 3).sum())\n",
    "count = (df['question_number'].value_counts() >= 2).sum()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "96467b8e-8cc9-4c70-a38a-d12c6d971cd8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "421"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f469b40b-ff0b-4366-bc5c-e676cc29cef1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_name\n",
       "bryanchrist                           1925\n",
       "kidleader                              149\n",
       "Raceaturtle                            134\n",
       "not-logged-in-d686d4334d00d113492a      23\n",
       "not-logged-in-8a7e887ccc7df8e54aff      18\n",
       "not-logged-in-a995833e0064b1a62f8a      16\n",
       "not-logged-in-21dd56b989228e883dee      12\n",
       "not-logged-in-01d1a4de2519268b1480       9\n",
       "not-logged-in-54d52b93eaf28770678a       8\n",
       "not-logged-in-32c214cfab64a34bb50f       7\n",
       "not-logged-in-c87d545dfc280e8ea529       7\n",
       "not-logged-in-2627d903c7ae22fddac0       6\n",
       "not-logged-in-a639c542b133d230733d       6\n",
       "not-logged-in-d43d96c64ff9f31ea237       5\n",
       "not-logged-in-c89b428b819bcea962db       5\n",
       "not-logged-in-f6e35db41dff1f893344       5\n",
       "not-logged-in-16c3a563e4df628d4ce6       5\n",
       "not-logged-in-695a0b07f6c9a0239266       5\n",
       "not-logged-in-1177246b7a4084f7c5af       5\n",
       "not-logged-in-91ae611983ee4e790f42       5\n",
       "not-logged-in-4d1e22248ff8d5ac8526       4\n",
       "cate11                                   4\n",
       "not-logged-in-419b4c84ee0ddc826ec3       4\n",
       "not-logged-in-295dea9e2f5b8feca6a9       4\n",
       "not-logged-in-4434f482a6b3f09f0b29       4\n",
       "not-logged-in-207d334a2eae31af9301       3\n",
       "not-logged-in-26b3f83f9405b275bde3       3\n",
       "not-logged-in-254f1b0adc11b05dfe9c       2\n",
       "not-logged-in-9cfcf496ebf1b813f19b       2\n",
       "not-logged-in-56a3d371332e2b539a25       2\n",
       "not-logged-in-3d368448749e6fc7f07d       1\n",
       "not-logged-in-5a847c15a3bd5c1659fe       1\n",
       "not-logged-in-726cbb6c1fb6ea8719ad       1\n",
       "not-logged-in-329ddf4b8909579aafd1       1\n",
       "not-logged-in-263b2fed971713715cec       1\n",
       "not-logged-in-95af40fb4ea11242ed23       1\n",
       "not-logged-in-ab81d6eefed8fef91f6b       1\n",
       "not-logged-in-02c614ded7364b0426f2       1\n",
       "not-logged-in-14a4e4cc131dfa6862b1       1\n",
       "not-logged-in-1f62abf888da440851e4       1\n",
       "not-logged-in-fb027728d9aacb480a0d       1\n",
       "not-logged-in-5f8bdba5be391d2962a3       1\n",
       "not-logged-in-80cd4372150b2f3e7a80       1\n",
       "not-logged-in-ed28403e8e1a346cff41       1\n",
       "not-logged-in-1d91c43026bd00c462ab       1\n",
       "not-logged-in-bdc2477f3a5c3e6096c4       1\n",
       "not-logged-in-e8a2b4e0a2577d5a1dbd       1\n",
       "not-logged-in-c81757e666dc6bd0b749       1\n",
       "not-logged-in-8ed8477c6dd7b3ad4a76       1\n",
       "not-logged-in-0f1f8588405680b90eee       1\n",
       "not-logged-in-af0b3f211a07cd9fa3f5       1\n",
       "not-logged-in-8d8a50a5ab4ff2a4c782       1\n",
       "not-logged-in-64f847afc8766ebae3cf       1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['user_name'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e0f7b039-4dbd-442e-8d20-fa08c6ba35ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4201514617.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['average_solvability'] = df.groupby('question_number')['solvability'].transform('mean')\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4201514617.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['average_accuracy'] = df.groupby('question_number')['accuracy'].transform('mean')\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4201514617.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['average_appropriateness']=df.groupby('question_number')['appropriateness'].transform('mean')\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/4201514617.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['average_good']=df.groupby('question_number')['good'].transform('mean')\n"
     ]
    }
   ],
   "source": [
    "# 1. Group by question and calculate the average score\n",
    "df['average_solvability'] = df.groupby('question_number')['solvability'].transform('mean')\n",
    "df['average_accuracy'] = df.groupby('question_number')['accuracy'].transform('mean')\n",
    "df['average_appropriateness']=df.groupby('question_number')['appropriateness'].transform('mean')\n",
    "df['average_good']=df.groupby('question_number')['good'].transform('mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8925a753-197c-4de4-a3f7-2899f514e87e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6769596199524941 0.04467091124778158\n"
     ]
    }
   ],
   "source": [
    "# Calculate overall agreement\n",
    "questions = []\n",
    "disagree_list = []\n",
    "for i in range(0, len(df)):\n",
    "    question = df.iloc[i]['question_number']\n",
    "    if df.iloc[i]['average_solvability']>0 and df.iloc[i]['average_solvability']<1:\n",
    "        if question not in questions:\n",
    "            questions.append(question)\n",
    "        disagree_list.append({'model': df.iloc[i]['model'], 'question': question, 'reviewer': df.iloc[i]['user_name'], 'rating': df.iloc[i]['solvability'], \n",
    "                              'average_score': df.iloc[i]['average_solvability'], 'criteria': 'solvability'})\n",
    "    if df.iloc[i]['average_accuracy']>0 and df.iloc[i]['average_accuracy']<1:\n",
    "        if question not in questions:\n",
    "            questions.append(question)\n",
    "        disagree_list.append({'model': df.iloc[i]['model'], 'question': question, 'reviewer': df.iloc[i]['user_name'], 'rating': df.iloc[i]['accuracy'], \n",
    "                              'average_score': df.iloc[i]['average_accuracy'], 'criteria': 'accuracy'})\n",
    "    if df.iloc[i]['average_appropriateness']>0 and df.iloc[i]['average_appropriateness']<1:\n",
    "        if question not in questions:\n",
    "            questions.append(question)\n",
    "        disagree_list.append({'model': df.iloc[i]['model'], 'question': question, 'reviewer': df.iloc[i]['user_name'], 'rating': df.iloc[i]['appropriateness'], \n",
    "                              'average_score': df.iloc[i]['average_appropriateness'], 'criteria': 'appropriateness'})\n",
    "unique_list = list(set(questions))\n",
    "proportion_unique = 1 - len(unique_list)/count\n",
    "\n",
    "# Calculate the standard error\n",
    "standard_error = math.sqrt((proportion_unique * (1 - proportion_unique)) / count)\n",
    "\n",
    "# Z-score for a 95% confidence interval\n",
    "z_score = 1.96\n",
    "\n",
    "# Calculate the margin of error\n",
    "margin_of_error = z_score * standard_error\n",
    "print(proportion_unique, margin_of_error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "b2758a3a-c1ae-4cab-8d1d-bf6575bb6ddf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>question</th>\n",
       "      <th>reviewer</th>\n",
       "      <th>rating</th>\n",
       "      <th>average_score</th>\n",
       "      <th>criteria</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>llema</td>\n",
       "      <td>llema_q801</td>\n",
       "      <td>not-logged-in-329ddf4b8909579aafd1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>solvability</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>llema</td>\n",
       "      <td>llema_q526</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>solvability</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mammoth</td>\n",
       "      <td>mammoth_q129</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>llama</td>\n",
       "      <td>llama_q706</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>accuracy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mammoth</td>\n",
       "      <td>mammoth_q261</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>304</th>\n",
       "      <td>gpt35</td>\n",
       "      <td>gpt35_q73</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317</th>\n",
       "      <td>gpt4</td>\n",
       "      <td>gpt4_q233</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>gpt4</td>\n",
       "      <td>gpt4_q225</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>319</th>\n",
       "      <td>gpt4</td>\n",
       "      <td>gpt4_q235</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>gpt4</td>\n",
       "      <td>gpt4_q81</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>solvability</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>147 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       model      question                            reviewer  rating  \\\n",
       "0      llema    llema_q801  not-logged-in-329ddf4b8909579aafd1     1.0   \n",
       "1      llema    llema_q526                         bryanchrist     0.0   \n",
       "2    mammoth  mammoth_q129                         bryanchrist     0.0   \n",
       "3      llama    llama_q706                         bryanchrist     1.0   \n",
       "4    mammoth  mammoth_q261                         bryanchrist     0.0   \n",
       "..       ...           ...                                 ...     ...   \n",
       "304    gpt35     gpt35_q73                         bryanchrist     1.0   \n",
       "317     gpt4     gpt4_q233                         bryanchrist     1.0   \n",
       "318     gpt4     gpt4_q225                         bryanchrist     0.0   \n",
       "319     gpt4     gpt4_q235                         bryanchrist     0.0   \n",
       "329     gpt4      gpt4_q81                         bryanchrist     1.0   \n",
       "\n",
       "     average_score         criteria  \n",
       "0         0.500000      solvability  \n",
       "1         0.333333      solvability  \n",
       "2         0.500000  appropriateness  \n",
       "3         0.500000         accuracy  \n",
       "4         0.500000  appropriateness  \n",
       "..             ...              ...  \n",
       "304       0.500000  appropriateness  \n",
       "317       0.500000  appropriateness  \n",
       "318       0.500000  appropriateness  \n",
       "319       0.500000  appropriateness  \n",
       "329       0.500000      solvability  \n",
       "\n",
       "[147 rows x 6 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disagree_list = pd.DataFrame.from_dict(disagree_list)\n",
    "disagree_list = disagree_list.drop_duplicates(subset = ['question', 'criteria'])\n",
    "disagree_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b1ec76e7-fbdb-457e-b67f-b3df33eebfac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "criteria\n",
       "appropriateness    82\n",
       "solvability        46\n",
       "accuracy           19\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disagree_list['criteria'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a9ae52a0-4236-4e7c-9817-3cf3b75609eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model\n",
       "llema       48\n",
       "mammoth     37\n",
       "llama       23\n",
       "gpt35       18\n",
       "mathwell    17\n",
       "gpt4         4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disagree_list['model'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "9210d442-ebc3-4683-be6b-a648d3717107",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>question</th>\n",
       "      <th>reviewer</th>\n",
       "      <th>rating</th>\n",
       "      <th>average_score</th>\n",
       "      <th>criteria</th>\n",
       "      <th>over_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>llema</td>\n",
       "      <td>llema_q801</td>\n",
       "      <td>not-logged-in-329ddf4b8909579aafd1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>solvability</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>llema</td>\n",
       "      <td>llema_q526</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>solvability</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mammoth</td>\n",
       "      <td>mammoth_q129</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>llama</td>\n",
       "      <td>llama_q706</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mammoth</td>\n",
       "      <td>mammoth_q261</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>304</th>\n",
       "      <td>gpt35</td>\n",
       "      <td>gpt35_q73</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317</th>\n",
       "      <td>gpt4</td>\n",
       "      <td>gpt4_q233</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>gpt4</td>\n",
       "      <td>gpt4_q225</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>319</th>\n",
       "      <td>gpt4</td>\n",
       "      <td>gpt4_q235</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>appropriateness</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>gpt4</td>\n",
       "      <td>gpt4_q81</td>\n",
       "      <td>bryanchrist</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>solvability</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>147 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       model      question                            reviewer  rating  \\\n",
       "0      llema    llema_q801  not-logged-in-329ddf4b8909579aafd1     1.0   \n",
       "1      llema    llema_q526                         bryanchrist     0.0   \n",
       "2    mammoth  mammoth_q129                         bryanchrist     0.0   \n",
       "3      llama    llama_q706                         bryanchrist     1.0   \n",
       "4    mammoth  mammoth_q261                         bryanchrist     0.0   \n",
       "..       ...           ...                                 ...     ...   \n",
       "304    gpt35     gpt35_q73                         bryanchrist     1.0   \n",
       "317     gpt4     gpt4_q233                         bryanchrist     1.0   \n",
       "318     gpt4     gpt4_q225                         bryanchrist     0.0   \n",
       "319     gpt4     gpt4_q235                         bryanchrist     0.0   \n",
       "329     gpt4      gpt4_q81                         bryanchrist     1.0   \n",
       "\n",
       "     average_score         criteria  over_score  \n",
       "0         0.500000      solvability           0  \n",
       "1         0.333333      solvability           0  \n",
       "2         0.500000  appropriateness           0  \n",
       "3         0.500000         accuracy           1  \n",
       "4         0.500000  appropriateness           0  \n",
       "..             ...              ...         ...  \n",
       "304       0.500000  appropriateness           1  \n",
       "317       0.500000  appropriateness           1  \n",
       "318       0.500000  appropriateness           0  \n",
       "319       0.500000  appropriateness           0  \n",
       "329       0.500000      solvability           1  \n",
       "\n",
       "[147 rows x 7 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def over_score(df):\n",
    "    if df['reviewer'] == 'bryanchrist':\n",
    "        score =  df['rating'] - df['average_score']\n",
    "        if score < 0: \n",
    "            return 0\n",
    "        if score > 0:\n",
    "            return 1\n",
    "    else: \n",
    "        score =  df['rating'] - df['average_score']\n",
    "        if score > 0: \n",
    "            return 0\n",
    "        if score < 0:\n",
    "            return 1\n",
    "disagree_list['over_score'] = disagree_list.apply(over_score, axis =1)\n",
    "disagree_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "6f1c4a73-1d19-43fd-9f8b-d50386d7c9c9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7959183673469388\n",
      "0.7058823529411764\n",
      "0.6956521739130435\n",
      "0.8541666666666666\n",
      "0.8918918918918919\n",
      "0.7222222222222222\n",
      "0.5\n"
     ]
    }
   ],
   "source": [
    "print(1 - disagree_list['over_score'].mean())\n",
    "print(1 - disagree_list[disagree_list['model']=='mathwell']['over_score'].mean())\n",
    "print(1 - disagree_list[disagree_list['model']=='llama']['over_score'].mean())\n",
    "print(1 - disagree_list[disagree_list['model']=='llema']['over_score'].mean())\n",
    "print(1 - disagree_list[disagree_list['model']=='mammoth']['over_score'].mean())\n",
    "print(1 - disagree_list[disagree_list['model']=='gpt35']['over_score'].mean())\n",
    "print(1 - disagree_list[disagree_list['model']=='gpt4']['over_score'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "402746a4-cfbd-48fd-8bee-10fdff3c7faa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 1.7184170009771527\n",
      "P-value: 0.09166997501209395\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "# Perform an independent two-sample t-test on percentage good\n",
    "t_statistic, p_value = stats.ttest_ind(disagree_list[disagree_list['model']=='mathwell']['over_score'], disagree_list[disagree_list['model']=='mammoth']['over_score'])\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "3980b2ae-91cd-463f-a0ce-1078e7732636",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8907363420427553 0.029800769428630733\n"
     ]
    }
   ],
   "source": [
    "#Solvability agreement\n",
    "questions = []\n",
    "for i in range(0, len(df)):\n",
    "    question = df.iloc[i]['question_number']\n",
    "    if df.iloc[i]['average_solvability']>0 and df.iloc[i]['average_solvability']<1:\n",
    "        if question not in questions:\n",
    "            questions.append(question)\n",
    "unique_list = list(set(questions))\n",
    "proportion_unique = 1 - len(unique_list)/count\n",
    "\n",
    "# Calculate the standard error\n",
    "standard_error = math.sqrt((proportion_unique * (1 - proportion_unique)) / count)\n",
    "\n",
    "# Z-score for a 95% confidence interval\n",
    "z_score = 1.96\n",
    "\n",
    "# Calculate the margin of error\n",
    "margin_of_error = z_score * standard_error\n",
    "print(proportion_unique, margin_of_error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "000d66e0-bf1a-44e7-abe5-ec2fab7a551c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8052256532066508 0.037830292819648316\n"
     ]
    }
   ],
   "source": [
    "# appropriateness agreement\n",
    "questions = []\n",
    "for i in range(0, len(df)):\n",
    "    question = df.iloc[i]['question_number']\n",
    "    if df.iloc[i]['average_appropriateness']>0 and df.iloc[i]['average_appropriateness']<1:\n",
    "        if question not in questions:\n",
    "            questions.append(question)\n",
    "unique_list = list(set(questions))\n",
    "proportion_unique = 1 - len(unique_list)/count\n",
    "\n",
    "# Calculate the standard error\n",
    "standard_error = math.sqrt((proportion_unique * (1 - proportion_unique)) / count)\n",
    "\n",
    "# Z-score for a 95% confidence interval\n",
    "z_score = 1.96\n",
    "\n",
    "# Calculate the margin of error\n",
    "margin_of_error = z_score * standard_error\n",
    "print(proportion_unique, margin_of_error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "45e33827-6b9d-4282-b28a-57bf09de6c59",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9548693586698337 0.01983000184541989\n"
     ]
    }
   ],
   "source": [
    "# accuracy agreement\n",
    "questions = []\n",
    "for i in range(0, len(df)):\n",
    "    question = df.iloc[i]['question_number']\n",
    "    if df.iloc[i]['average_accuracy']>0 and df.iloc[i]['average_accuracy']<1:\n",
    "        if question not in questions:\n",
    "            questions.append(question)\n",
    "unique_list = list(set(questions))\n",
    "proportion_unique = 1 - len(unique_list)/count\n",
    "\n",
    "# Calculate the standard error\n",
    "standard_error = math.sqrt((proportion_unique * (1 - proportion_unique)) / count)\n",
    "\n",
    "# Z-score for a 95% confidence interval\n",
    "z_score = 1.96\n",
    "\n",
    "# Calculate the margin of error\n",
    "margin_of_error = z_score * standard_error\n",
    "print(proportion_unique, margin_of_error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "bec7b352-742d-4bfe-af26-367ccaabf8d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7814726840855106 0.03947524101402549\n"
     ]
    }
   ],
   "source": [
    "#mac agreement\n",
    "questions = []\n",
    "for i in range(0, len(df)):\n",
    "    question = df.iloc[i]['question_number']\n",
    "    if df.iloc[i]['average_good']>0 and df.iloc[i]['average_good']<1:\n",
    "        if question not in questions:\n",
    "            questions.append(question)\n",
    "unique_list = list(set(questions))\n",
    "proportion_unique = 1 - len(unique_list)/count\n",
    "\n",
    "# Calculate the standard error\n",
    "standard_error = math.sqrt((proportion_unique * (1 - proportion_unique)) / count)\n",
    "\n",
    "# Z-score for a 95% confidence interval\n",
    "z_score = 1.96\n",
    "\n",
    "# Calculate the margin of error\n",
    "margin_of_error = z_score * standard_error\n",
    "print(proportion_unique, margin_of_error)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "961bd1d0-32e0-4ee7-bcf6-b6afd2c1b82f",
   "metadata": {},
   "source": [
    "## Retain Necessary Columns and Evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "bc34e432-5b03-46f2-91ad-0522288bf824",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def accuracy(df):\n",
    "#     if df['average_accuracy']>.5:\n",
    "#         return 1\n",
    "#     if df['average_accuracy']==np.nan:\n",
    "#         return np.nan\n",
    "#     if df['average_accuracy']<=0.5:\n",
    "#         return 0\n",
    "# def solvability(df):\n",
    "#     if df['average_solvability']>.5:\n",
    "#         return 1\n",
    "#     else:\n",
    "#         return 0\n",
    "# def appropriateness(df):\n",
    "#     if df['average_appropriateness']>.5:\n",
    "#         return 1\n",
    "#     else:\n",
    "#         return 0\n",
    "# def good(df):\n",
    "#     if df['average_good']>.5:\n",
    "#         return 1\n",
    "#     else:\n",
    "#         return 0\n",
    "# df['solvability']=df.apply(solvability, axis=1)\n",
    "# df['appropriateness']=df.apply(appropriateness, axis=1)\n",
    "# df['accuracy']=df.apply(accuracy, axis=1)\n",
    "# df['good'] = df.apply(good, axis=1)\n",
    "# comment above this out\n",
    "df = df[df['user_name']=='bryanchrist']\n",
    "df = df[['model','question_number', 'solvability', 'accuracy', 'appropriateness', 'topic', 'operations','notes']]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "535394b5-17f6-41fc-8ff5-c186bc357269",
   "metadata": {},
   "source": [
    "## Calculate Operations Variables and Overall Indicator Variable for Question Being Good in All Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "857dd98a-4164-4dd4-8d23-6feabf1501b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>question_number</th>\n",
       "      <th>solvability</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>appropriateness</th>\n",
       "      <th>topic</th>\n",
       "      <th>operations</th>\n",
       "      <th>notes</th>\n",
       "      <th>addition</th>\n",
       "      <th>subtraction</th>\n",
       "      <th>multiplication</th>\n",
       "      <th>division</th>\n",
       "      <th>fractions</th>\n",
       "      <th>decimals</th>\n",
       "      <th>no_ops</th>\n",
       "      <th>total_ops</th>\n",
       "      <th>good</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>llama</td>\n",
       "      <td>llama_q1053</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>[Subtraction, Addition]</td>\n",
       "      <td></td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>mammoth</td>\n",
       "      <td>mammoth_q127</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>[Multiplication]</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>llama</td>\n",
       "      <td>llama_q1061</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>[Division, Decimals]</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>llema</td>\n",
       "      <td>llema_q526</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NA</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>mathwell</td>\n",
       "      <td>mathwell_q528</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>[Addition]</td>\n",
       "      <td></td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2485</th>\n",
       "      <td>egsm_only</td>\n",
       "      <td>egsm_only_q577</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>[Addition]</td>\n",
       "      <td></td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2486</th>\n",
       "      <td>egsm_only</td>\n",
       "      <td>egsm_only_q434</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>[Addition, Subtraction]</td>\n",
       "      <td></td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2487</th>\n",
       "      <td>egsm_only</td>\n",
       "      <td>egsm_only_q177</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>[Addition, Subtraction]</td>\n",
       "      <td></td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2488</th>\n",
       "      <td>egsm_only</td>\n",
       "      <td>egsm_only_q253</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>[Subtraction, Multiplication]</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2489</th>\n",
       "      <td>egsm_only</td>\n",
       "      <td>egsm_only_q325</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NA</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1925 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          model question_number  solvability  accuracy  appropriateness  \\\n",
       "0         llama     llama_q1053            1       1.0              1.0   \n",
       "9       mammoth    mammoth_q127            1       1.0              1.0   \n",
       "10        llama     llama_q1061            1       0.0              1.0   \n",
       "11        llema      llema_q526            0       NaN              NaN   \n",
       "12     mathwell   mathwell_q528            1       1.0              1.0   \n",
       "...         ...             ...          ...       ...              ...   \n",
       "2485  egsm_only  egsm_only_q577            1       1.0              1.0   \n",
       "2486  egsm_only  egsm_only_q434            1       1.0              0.0   \n",
       "2487  egsm_only  egsm_only_q177            1       1.0              1.0   \n",
       "2488  egsm_only  egsm_only_q253            1       1.0              0.0   \n",
       "2489  egsm_only  egsm_only_q325            0       NaN              NaN   \n",
       "\n",
       "      topic                     operations notes  addition  subtraction  \\\n",
       "0         1        [Subtraction, Addition]             1.0          1.0   \n",
       "9         1               [Multiplication]             0.0          0.0   \n",
       "10        1           [Division, Decimals]             0.0          0.0   \n",
       "11        1                             NA             NaN          NaN   \n",
       "12        1                     [Addition]             1.0          0.0   \n",
       "...     ...                            ...   ...       ...          ...   \n",
       "2485      1                     [Addition]             1.0          0.0   \n",
       "2486      1        [Addition, Subtraction]             1.0          1.0   \n",
       "2487      1        [Addition, Subtraction]             1.0          1.0   \n",
       "2488      1  [Subtraction, Multiplication]             0.0          1.0   \n",
       "2489      1                             NA             NaN          NaN   \n",
       "\n",
       "      multiplication  division  fractions  decimals  no_ops  total_ops  good  \n",
       "0                0.0       0.0        0.0       0.0     0.0        2.0     1  \n",
       "9                1.0       0.0        0.0       0.0     0.0        1.0     1  \n",
       "10               0.0       1.0        0.0       1.0     0.0        2.0     0  \n",
       "11               NaN       NaN        NaN       NaN     NaN        NaN     0  \n",
       "12               0.0       0.0        0.0       0.0     0.0        1.0     1  \n",
       "...              ...       ...        ...       ...     ...        ...   ...  \n",
       "2485             0.0       0.0        0.0       0.0     0.0        1.0     1  \n",
       "2486             0.0       0.0        0.0       0.0     0.0        2.0     0  \n",
       "2487             0.0       0.0        0.0       0.0     0.0        2.0     1  \n",
       "2488             1.0       0.0        0.0       0.0     0.0        2.0     0  \n",
       "2489             NaN       NaN        NaN       NaN     NaN        NaN     0  \n",
       "\n",
       "[1925 rows x 17 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def addition(df):\n",
    "    if 'Addition' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def subtraction(df):\n",
    "    if 'Subtraction' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def multiplication(df):\n",
    "    if 'Multiplication' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def division(df):\n",
    "    if 'Division' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def fractions(df):\n",
    "    if 'Fractions' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def decimals(df):\n",
    "    if 'Decimals' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def none(df):\n",
    "    if 'Not Applicable: This question does not require a mathematical operation.' in df['operations']:\n",
    "        return 1\n",
    "    if df['operations']==\"NA\":\n",
    "        return np.nan\n",
    "    else: \n",
    "        return 0 \n",
    "def total_ops(df):\n",
    "    if df['no_ops']==1:\n",
    "        return np.nan\n",
    "    else:\n",
    "        return df['addition'] + df['subtraction'] + df['multiplication'] + df['decimals'] + df['division'] + df['fractions']\n",
    "df['addition'] = df.apply(addition, axis=1)\n",
    "df['subtraction'] = df.apply(subtraction, axis=1)\n",
    "df['multiplication'] = df.apply(multiplication, axis=1)\n",
    "df['division'] = df.apply(division, axis=1)\n",
    "df['fractions'] = df.apply(fractions, axis=1)\n",
    "df['decimals'] = df.apply(decimals, axis=1)\n",
    "df['no_ops'] = df.apply(none, axis=1)\n",
    "df['total_ops'] = df.apply(total_ops, axis=1)\n",
    "def good(df):\n",
    "    if df['solvability']==1:\n",
    "        if df['accuracy']==1:\n",
    "            if df['no_ops']==0:\n",
    "                if df['appropriateness']==1:\n",
    "                   # if df['topic']==1:\n",
    "                    return 1\n",
    "    if df['accuracy']==np.nan:\n",
    "        return 0\n",
    "    if df['no_ops']==np.nan:\n",
    "        return 0\n",
    "    if df['appropriateness']==np.nan:\n",
    "        return 0\n",
    "    else:\n",
    "        return 0\n",
    "df['good'] = df.apply(good, axis = 1)\n",
    "def good_recode(df):\n",
    "    if df['good']==np.nan:\n",
    "        return 0\n",
    "    else:\n",
    "        return df['good']\n",
    "df['good'] = df.apply(good_recode, axis=1)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "b2991b8a-0d8a-4a8d-9f3c-572cb413f8aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matched\n",
       "both          1850\n",
       "left_only        0\n",
       "right_only       0\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop(['operations'], axis=1)\n",
    "df = df.drop_duplicates()\n",
    "full_data = pd.read_csv(\"data/zooniverse_model_questions.csv\")\n",
    "finetune = pd.read_csv('data/finetuning_data.csv')\n",
    "full_data = full_data[~full_data['question'].isin(finetune['question'])]\n",
    "llama_gsm_qs = pd.read_csv(\"data/llama_gsm_questions.csv\")\n",
    "egsm_only_qs = pd.read_csv(\"data/egsm_only_questions.csv\")\n",
    "full_data = pd.concat([full_data, llama_gsm_qs, egsm_only_qs])                           \n",
    "df = pd.merge(df, full_data,\n",
    "              on = 'question_number',\n",
    "              how='inner',\n",
    "              validate='one_to_one', \n",
    "              indicator = 'matched')\n",
    "df['matched'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "b4ace228-ea19-4f60-ae6d-0ca823a77977",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>question_number</th>\n",
       "      <th>solvability</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>appropriateness</th>\n",
       "      <th>topic</th>\n",
       "      <th>notes</th>\n",
       "      <th>addition</th>\n",
       "      <th>subtraction</th>\n",
       "      <th>multiplication</th>\n",
       "      <th>...</th>\n",
       "      <th>Unnamed: 0.2</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic_y</th>\n",
       "      <th>Unnamed: 0.4</th>\n",
       "      <th>Unnamed: 0.5</th>\n",
       "      <th>matched</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>llama</td>\n",
       "      <td>llama_q1053</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>54.0</td>\n",
       "      <td>570.0</td>\n",
       "      <td>1062.0</td>\n",
       "      <td>Steph Curry has 3000 points. 1000 of them are ...</td>\n",
       "      <td>def solution():\\n    #Steph Curry has 3000 poi...</td>\n",
       "      <td>0</td>\n",
       "      <td>Steph Curry</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mammoth</td>\n",
       "      <td>mammoth_q127</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>673.0</td>\n",
       "      <td>652.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>The soccer team has 15 players. Each player ha...</td>\n",
       "      <td>def solution():\\n    #Number of players\\n    p...</td>\n",
       "      <td>30</td>\n",
       "      <td>soccer</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>llama</td>\n",
       "      <td>llama_q1061</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>47.0</td>\n",
       "      <td>809.0</td>\n",
       "      <td>1070.0</td>\n",
       "      <td>A commercial airplane takes 12 hours to fly fr...</td>\n",
       "      <td>def solution():\\n    #A commercial airplane ta...</td>\n",
       "      <td>0.6</td>\n",
       "      <td>planes</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>llema</td>\n",
       "      <td>llema_q526</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1126.0</td>\n",
       "      <td>863.0</td>\n",
       "      <td>526.0</td>\n",
       "      <td>In the original comic book series, the four Tu...</td>\n",
       "      <td>def solution():\\n    #Raphael is 6 inches tall...</td>\n",
       "      <td>30</td>\n",
       "      <td>Ninja Turtles</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>llema</td>\n",
       "      <td>llema_q669</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>467.0</td>\n",
       "      <td>280.0</td>\n",
       "      <td>671.0</td>\n",
       "      <td>Spiderman has a web-shooter that shoots spider...</td>\n",
       "      <td>def solution():\\n    #Spiderman needs to load ...</td>\n",
       "      <td>180</td>\n",
       "      <td>Spiderman</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>egsm_only</td>\n",
       "      <td>egsm_only_q221</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>285.0</td>\n",
       "      <td>A volleyball team has 1200 points. 1000 of the...</td>\n",
       "      <td>def solution():\\n    #The volleyball team has ...</td>\n",
       "      <td>0</td>\n",
       "      <td>volleyball</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>egsm_only</td>\n",
       "      <td>egsm_only_q97</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>121.0</td>\n",
       "      <td>A dog has 12 bones. 4 of the bones are in the ...</td>\n",
       "      <td>def solution():\\n    #A dog has 12 bones\\n    ...</td>\n",
       "      <td>0</td>\n",
       "      <td>dogs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>egsm_only</td>\n",
       "      <td>egsm_only_q79</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.0</td>\n",
       "      <td>The Cedar Rapids Volleyball team has 12 player...</td>\n",
       "      <td>def solution():\\n    #Total number of players\\...</td>\n",
       "      <td>0</td>\n",
       "      <td>volleyball</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>egsm_only</td>\n",
       "      <td>egsm_only_q220</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>284.0</td>\n",
       "      <td>The T-Rex is the largest dinosaur.  It is 1000...</td>\n",
       "      <td>def solution():\\n    #The T-Rex is 100000 time...</td>\n",
       "      <td>100000000</td>\n",
       "      <td>dinosaurs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>egsm_only</td>\n",
       "      <td>egsm_only_q127</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>158.0</td>\n",
       "      <td>A volleyball team has 12 players. 3 of the pla...</td>\n",
       "      <td>def solution():\\n    #A volleyball team has 12...</td>\n",
       "      <td>0</td>\n",
       "      <td>volleyball</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1850 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        model question_number  solvability  accuracy  appropriateness  topic  \\\n",
       "0       llama     llama_q1053            1       1.0              1.0      1   \n",
       "1     mammoth    mammoth_q127            1       1.0              1.0      1   \n",
       "2       llama     llama_q1061            1       0.0              1.0      1   \n",
       "3       llema      llema_q526            0       NaN              NaN      1   \n",
       "5       llema      llema_q669            1       1.0              1.0      1   \n",
       "..        ...             ...          ...       ...              ...    ...   \n",
       "95  egsm_only  egsm_only_q221            0       NaN              NaN      1   \n",
       "96  egsm_only   egsm_only_q97            0       NaN              NaN      1   \n",
       "97  egsm_only   egsm_only_q79            0       NaN              NaN      1   \n",
       "98  egsm_only  egsm_only_q220            0       NaN              NaN      1   \n",
       "99  egsm_only  egsm_only_q127            0       NaN              NaN      1   \n",
       "\n",
       "   notes  addition  subtraction  multiplication  ...  Unnamed: 0.2  \\\n",
       "0              1.0          1.0             0.0  ...          54.0   \n",
       "1              0.0          0.0             1.0  ...         673.0   \n",
       "2              0.0          0.0             0.0  ...          47.0   \n",
       "3              NaN          NaN             NaN  ...        1126.0   \n",
       "5              0.0          1.0             0.0  ...         467.0   \n",
       "..   ...       ...          ...             ...  ...           ...   \n",
       "95   NaN       NaN          NaN             NaN  ...           NaN   \n",
       "96   NaN       NaN          NaN             NaN  ...           NaN   \n",
       "97   NaN       NaN          NaN             NaN  ...           NaN   \n",
       "98   NaN       NaN          NaN             NaN  ...           NaN   \n",
       "99   NaN       NaN          NaN             NaN  ...           NaN   \n",
       "\n",
       "    Unnamed: 0.1  Unnamed: 0  \\\n",
       "0          570.0      1062.0   \n",
       "1          652.0       127.0   \n",
       "2          809.0      1070.0   \n",
       "3          863.0       526.0   \n",
       "5          280.0       671.0   \n",
       "..           ...         ...   \n",
       "95           NaN       285.0   \n",
       "96           NaN       121.0   \n",
       "97           NaN       100.0   \n",
       "98           NaN       284.0   \n",
       "99           NaN       158.0   \n",
       "\n",
       "                                             question  \\\n",
       "0   Steph Curry has 3000 points. 1000 of them are ...   \n",
       "1   The soccer team has 15 players. Each player ha...   \n",
       "2   A commercial airplane takes 12 hours to fly fr...   \n",
       "3   In the original comic book series, the four Tu...   \n",
       "5   Spiderman has a web-shooter that shoots spider...   \n",
       "..                                                ...   \n",
       "95  A volleyball team has 1200 points. 1000 of the...   \n",
       "96  A dog has 12 bones. 4 of the bones are in the ...   \n",
       "97  The Cedar Rapids Volleyball team has 12 player...   \n",
       "98  The T-Rex is the largest dinosaur.  It is 1000...   \n",
       "99  A volleyball team has 12 players. 3 of the pla...   \n",
       "\n",
       "                                             solution     answer  \\\n",
       "0   def solution():\\n    #Steph Curry has 3000 poi...          0   \n",
       "1   def solution():\\n    #Number of players\\n    p...         30   \n",
       "2   def solution():\\n    #A commercial airplane ta...        0.6   \n",
       "3   def solution():\\n    #Raphael is 6 inches tall...         30   \n",
       "5   def solution():\\n    #Spiderman needs to load ...        180   \n",
       "..                                                ...        ...   \n",
       "95  def solution():\\n    #The volleyball team has ...          0   \n",
       "96  def solution():\\n    #A dog has 12 bones\\n    ...          0   \n",
       "97  def solution():\\n    #Total number of players\\...          0   \n",
       "98  def solution():\\n    #The T-Rex is 100000 time...  100000000   \n",
       "99  def solution():\\n    #A volleyball team has 12...          0   \n",
       "\n",
       "          topic_y  Unnamed: 0.4  Unnamed: 0.5 matched  \n",
       "0     Steph Curry           NaN           NaN     NaN  \n",
       "1          soccer           NaN           NaN     NaN  \n",
       "2          planes           NaN           NaN     NaN  \n",
       "3   Ninja Turtles           NaN           NaN     NaN  \n",
       "5       Spiderman           NaN           NaN     NaN  \n",
       "..            ...           ...           ...     ...  \n",
       "95     volleyball           NaN           NaN    both  \n",
       "96           dogs           NaN           NaN    both  \n",
       "97     volleyball           NaN           NaN    both  \n",
       "98      dinosaurs           NaN           NaN    both  \n",
       "99     volleyball           NaN           NaN    both  \n",
       "\n",
       "[1850 rows x 27 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df = pd.read_csv('data/filtered_models.csv')\n",
    "df = df.rename({'topic_x': 'topic'}, axis =1)\n",
    "df = df.drop('matched', axis = 1)\n",
    "df = df[df['model']!= 'mathwell']\n",
    "df = df[df['model']!='gpt35']\n",
    "df = df[df['model']!='gpt4']\n",
    "egsm = df[df['model']=='egsm_only']\n",
    "df = df[df['model']!='egsm_only']\n",
    "mathwell = pd.read_csv('data/mathwell_final_annotations.csv')\n",
    "gpt35 = pd.read_csv('data/gpt35_annotations.csv')\n",
    "gpt4 = pd.read_csv('data/gpt4_annotations.csv')\n",
    "egsm_only = pd.read_csv('data/egsm_only_annotations_final.csv')\n",
    "egsm = egsm[~egsm['question'].isin(egsm_only['question'])]\n",
    "egsm.loc[egsm['question_number'] == 'egsm_only_q44', 'appropriateness'] = 1\n",
    "egsm_only = pd.concat([egsm, egsm_only])\n",
    "df = pd.concat([df, mathwell, gpt35, gpt4, egsm_only])\n",
    "def good(df):\n",
    "    if df['solvability']==1:\n",
    "        if df['accuracy']==1:\n",
    "            if df['no_ops']==0:\n",
    "                if df['appropriateness']==1:\n",
    "                   # if df['topic']==1:\n",
    "                    return 1\n",
    "    if df['accuracy']==np.nan:\n",
    "        return 0\n",
    "    if df['no_ops']==np.nan:\n",
    "        return 0\n",
    "    if df['appropriateness']==np.nan:\n",
    "        return 0\n",
    "    else:\n",
    "        return 0\n",
    "df['good'] = df.apply(good, axis = 1)\n",
    "def good_recode(df):\n",
    "    if df['good']==np.nan:\n",
    "        return 0\n",
    "    else:\n",
    "        return df['good']\n",
    "df['good'] = df.apply(good_recode, axis=1)\n",
    "merged = df\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "163a4788-281a-464e-961a-ae120352e246",
   "metadata": {},
   "source": [
    "## Calculate Average Statistics for Each Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "2171c921-f750-4711-b961-8355e3a21aeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "llama = df[df['model']=='llama']\n",
    "llema = df[df['model']=='llema']\n",
    "mathwell = df[df['model']=='mathwell']\n",
    "#mathwell.to_csv('data/mathwell_final_annotations.csv')\n",
    "mammoth = df[df['model']=='mammoth']\n",
    "gpt35 = df[df['model']=='gpt35']\n",
    "#gpt35.to_csv('data/gpt35_annotations.csv')\n",
    "gpt4 = df[df['model']=='gpt4']\n",
    "#gpt4.to_csv('data/gpt4_annotations.csv')\n",
    "llama_gsm = df[df['model']=='llama_gsm']\n",
    "egsm_only = df[df['model']=='egsm_only']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "ea7cd071-50f4-4d4a-b1a4-f12efcec681a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpt4 metrics:\n",
      "Solvability: 0.948 Standard Deviation: 0.22202702538204672 Standard Error: 1.407039102564168\n",
      "Accuracy: 0.9578059071729957 Standard Deviation: 0.20103171729234767 Standard Error: 1.3086050173447799\n",
      "Appropriateness: 0.8438818565400844 Standard Deviation: 0.3629673108456804 Standard Error: 2.362715946031866\n",
      "Percentage about Requested Topic: 0.992 Standard Deviation: 0.08908422980528033 Standard Error: 0.5645483676690164\n",
      "Percentage Good in All Metrics: 0.788 Standard Deviation: 0.40872484632084705 Standard Error: 2.590188469054112\n",
      "Percentage good with executable code: 0.5260209424083769 Standard Error: 1.7290525120649176\n"
     ]
    }
   ],
   "source": [
    "solvability = gpt4['solvability'].mean()\n",
    "solvability_int = st.t.interval(confidence=0.95, df=len(gpt4['solvability'])-1, loc=np.mean(gpt4['solvability']), scale=st.sem(gpt4['solvability']))\n",
    "solvability_se = st.sem(gpt4['solvability'])\n",
    "solvability_sd = np.std(gpt4['solvability'])\n",
    "accuracy = gpt4['accuracy'].mean()\n",
    "accuracy_int = st.t.interval(confidence=0.95, df=len(gpt4['accuracy'].dropna())-1, loc=np.mean(gpt4['accuracy'].dropna()), scale=st.sem(gpt4['accuracy'].dropna()))\n",
    "accuracy_sd = np.std(gpt4['accuracy'])\n",
    "accuracy_se = st.sem(gpt4['accuracy'].dropna())\n",
    "appropriateness = gpt4['appropriateness'].mean()\n",
    "appropriateness_int = st.t.interval(confidence=0.95, df=len(gpt4['appropriateness'].dropna())-1, loc=np.mean(gpt4['appropriateness'].dropna()), scale=st.sem(gpt4['appropriateness'].dropna()))\n",
    "appropriateness_sd = np.std(gpt4['appropriateness'])\n",
    "appropriateness_se = st.sem(gpt4['appropriateness'].dropna())\n",
    "topic = gpt4['topic'].mean()\n",
    "topic_int = st.t.interval(confidence=0.95, df=len(gpt4['topic'])-1, loc=np.mean(gpt4['topic']), scale=st.sem(gpt4['topic']))\n",
    "topic_sd = np.std(gpt4['topic'])\n",
    "topic_se = st.sem(gpt4['topic'])\n",
    "good = gpt4['good'].mean()\n",
    "good_int = st.t.interval(confidence=0.95, df=len(gpt4['good'])-1, loc=np.mean(gpt4['good']), scale=st.sem(gpt4['good']))\n",
    "good_sd = np.std(gpt4['good'])\n",
    "good_se = st.sem(gpt4['good'])\n",
    "execute = good * .6675392670157068\n",
    "execute_se = good_se * .6675392670157068\n",
    "print(f'''gpt4 metrics:\n",
    "Solvability: {solvability} Standard Deviation: {solvability_sd} Standard Error: {solvability_se*100}\n",
    "Accuracy: {accuracy} Standard Deviation: {accuracy_sd} Standard Error: {accuracy_se*100}\n",
    "Appropriateness: {appropriateness} Standard Deviation: {appropriateness_sd} Standard Error: {appropriateness_se*100}\n",
    "Percentage about Requested Topic: {topic} Standard Deviation: {topic_sd} Standard Error: {topic_se*100}\n",
    "Percentage Good in All Metrics: {good} Standard Deviation: {good_sd} Standard Error: {good_se*100}\n",
    "Percentage good with executable code: {execute} Standard Error: {execute_se*100}''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "9498e810-4708-4d9b-8b30-6137a749ce47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpt35 metrics:\n",
      "Solvability: 0.88 Standard Deviation: 0.32496153618543844 Standard Error: 2.059360059684\n",
      "Accuracy: 0.8954545454545455 Standard Deviation: 0.30596683231902577 Standard Error: 2.067532251484536\n",
      "Appropriateness: 0.7545454545454545 Standard Deviation: 0.4303563774016227 Standard Error: 2.908078902428723\n",
      "Percentage about Requested Topic: 0.988 Standard Deviation: 0.10888526071052963 Standard Error: 0.6900323023694277\n",
      "Percentage Good in All Metrics: 0.628 Standard Deviation: 0.48333839077813795 Standard Error: 3.063032594455827\n",
      "Percentage good with executable code: 0.6125680966264057 Standard Error: 2.9877644049210934\n"
     ]
    }
   ],
   "source": [
    "solvability = gpt35['solvability'].mean()\n",
    "solvability_int = st.t.interval(confidence=0.95, df=len(gpt35['solvability'])-1, loc=np.mean(gpt35['solvability']), scale=st.sem(gpt35['solvability']))\n",
    "solvability_se = st.sem(gpt35['solvability'])\n",
    "solvability_sd = np.std(gpt35['solvability'])\n",
    "accuracy = gpt35['accuracy'].mean()\n",
    "accuracy_int = st.t.interval(confidence=0.95, df=len(gpt35['accuracy'].dropna())-1, loc=np.mean(gpt35['accuracy'].dropna()), scale=st.sem(gpt35['accuracy'].dropna()))\n",
    "accuracy_sd = np.std(gpt35['accuracy'])\n",
    "accuracy_se = st.sem(gpt35['accuracy'].dropna())\n",
    "appropriateness = gpt35['appropriateness'].mean()\n",
    "appropriateness_int = st.t.interval(confidence=0.95, df=len(gpt35['appropriateness'].dropna())-1, loc=np.mean(gpt35['appropriateness'].dropna()), scale=st.sem(gpt35['appropriateness'].dropna()))\n",
    "appropriateness_sd = np.std(gpt35['appropriateness'])\n",
    "appropriateness_se = st.sem(gpt35['appropriateness'].dropna())\n",
    "topic = gpt35['topic'].mean()\n",
    "topic_int = st.t.interval(confidence=0.95, df=len(gpt35['topic'])-1, loc=np.mean(gpt35['topic']), scale=st.sem(gpt35['topic']))\n",
    "topic_sd = np.std(gpt35['topic'])\n",
    "topic_se = st.sem(gpt35['topic'])\n",
    "good = gpt35['good'].mean()\n",
    "good_int = st.t.interval(confidence=0.95, df=len(gpt35['good'])-1, loc=np.mean(gpt35['good']), scale=st.sem(gpt35['good']))\n",
    "good_sd = np.std(gpt35['good'])\n",
    "good_se = st.sem(gpt35['good'])\n",
    "execute = good * .9754269054560599\n",
    "execute_se = good_se * .9754269054560599\n",
    "print(f'''gpt35 metrics:\n",
    "Solvability: {solvability} Standard Deviation: {solvability_sd} Standard Error: {solvability_se*100}\n",
    "Accuracy: {accuracy} Standard Deviation: {accuracy_sd} Standard Error: {accuracy_se*100}\n",
    "Appropriateness: {appropriateness} Standard Deviation: {appropriateness_sd} Standard Error: {appropriateness_se*100}\n",
    "Percentage about Requested Topic: {topic} Standard Deviation: {topic_sd} Standard Error: {topic_se*100}\n",
    "Percentage Good in All Metrics: {good} Standard Deviation: {good_sd} Standard Error: {good_se*100}\n",
    "Percentage good with executable code: {execute} Standard Error: {execute_se*100}''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e0d72486-27a6-48e0-ab9b-4acd5c4820c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Llama metrics:\n",
      "Solvability: 0.84 Standard Deviation: 0.36660605559646725 Standard Error: 2.323271478206063\n",
      "Accuracy: 0.8952380952380953 Standard Deviation: 0.3062463845869243 Standard Error: 2.1183505372546803\n",
      "Appropriateness: 0.8095238095238095 Standard Deviation: 0.392676726249301 Standard Error: 2.7162017117022064\n",
      "Percentage about Requested Topic: 0.992 Standard Deviation: 0.08908422980528034 Standard Error: 0.5645483676690164\n",
      "Percentage Good in All Metrics: 0.624 Standard Deviation: 0.4843800161030593 Standard Error: 3.0696336267394577\n",
      "Percentage good with executable code: 0.3458576429404901 Standard Error: 1.7013721968509248\n"
     ]
    }
   ],
   "source": [
    "solvability = llama['solvability'].mean()\n",
    "solvability_int = st.t.interval(confidence=0.95, df=len(llama['solvability'])-1, loc=np.mean(llama['solvability']), scale=st.sem(llama['solvability']))\n",
    "solvability_se = st.sem(llama['solvability'])\n",
    "solvability_sd = np.std(llama['solvability'])\n",
    "accuracy = llama['accuracy'].mean()\n",
    "accuracy_int = st.t.interval(confidence=0.95, df=len(llama['accuracy'].dropna())-1, loc=np.mean(llama['accuracy'].dropna()), scale=st.sem(llama['accuracy'].dropna()))\n",
    "accuracy_sd = np.std(llama['accuracy'])\n",
    "accuracy_se = st.sem(llama['accuracy'].dropna())\n",
    "appropriateness = llama['appropriateness'].mean()\n",
    "appropriateness_int = st.t.interval(confidence=0.95, df=len(llama['appropriateness'].dropna())-1, loc=np.mean(llama['appropriateness'].dropna()), scale=st.sem(llama['appropriateness'].dropna()))\n",
    "appropriateness_sd = np.std(llama['appropriateness'])\n",
    "appropriateness_se = st.sem(llama['appropriateness'].dropna())\n",
    "topic = llama['topic'].mean()\n",
    "topic_int = st.t.interval(confidence=0.95, df=len(llama['topic'])-1, loc=np.mean(llama['topic']), scale=st.sem(llama['topic']))\n",
    "topic_sd = np.std(llama['topic'])\n",
    "topic_se = st.sem(llama['topic'])\n",
    "good = llama['good'].mean()\n",
    "good_int = st.t.interval(confidence=0.95, df=len(llama['good'])-1, loc=np.mean(llama['good']), scale=st.sem(llama['good']))\n",
    "good_sd = np.std(llama['good'])\n",
    "good_se = st.sem(llama['good'])\n",
    "execute = good * .55425904317386234\n",
    "execute_se = good_se * .55425904317386234\n",
    "print(f'''Llama metrics:\n",
    "Solvability: {solvability} Standard Deviation: {solvability_sd} Standard Error: {solvability_se*100}\n",
    "Accuracy: {accuracy} Standard Deviation: {accuracy_sd} Standard Error: {accuracy_se*100}\n",
    "Appropriateness: {appropriateness} Standard Deviation: {appropriateness_sd} Standard Error: {appropriateness_se*100}\n",
    "Percentage about Requested Topic: {topic} Standard Deviation: {topic_sd} Standard Error: {topic_se*100}\n",
    "Percentage Good in All Metrics: {good} Standard Deviation: {good_sd} Standard Error: {good_se*100}\n",
    "Percentage good with executable code: {execute} Standard Error: {execute_se*100}''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "e4b0a1c2-1f84-416e-b21d-971e2ef3101f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Llema metrics:\n",
      "Solvability: 0.488 Standard Deviation: 0.4998559792580259 Standard Error: 3.167708558254714\n",
      "Accuracy: 0.639344262295082 Standard Deviation: 0.48019077101235447 Standard Error: 4.365370645566858\n",
      "Appropriateness: 0.4180327868852459 Standard Deviation: 0.4932356191255864 Standard Error: 4.483960173868966\n",
      "Percentage about Requested Topic: 0.948 Standard Deviation: 0.2220270253820467 Standard Error: 1.4070391025641678\n",
      "Percentage Good in All Metrics: 0.152 Standard Deviation: 0.35902089075707005 Standard Error: 2.2752024491765463\n",
      "Percentage good with executable code: 0.036970791290493896 Standard Error: 0.5533949663955701\n"
     ]
    }
   ],
   "source": [
    "solvability = llema['solvability'].mean()\n",
    "solvability_int = st.t.interval(confidence=0.95, df=len(llema['solvability'])-1, loc=np.mean(llema['solvability']), scale=st.sem(llema['solvability']))\n",
    "solvability_sd = np.std(llema['solvability'])\n",
    "solvability_se = st.sem(llema['solvability'])\n",
    "accuracy = llema['accuracy'].mean()\n",
    "accuracy_int = st.t.interval(confidence=0.95, df=len(llema['accuracy'].dropna())-1, loc=np.mean(llema['accuracy'].dropna()), scale=st.sem(llema['accuracy'].dropna()))\n",
    "accuracy_sd = np.std(llema['accuracy'])\n",
    "accuracy_se = st.sem(llema['accuracy'].dropna())\n",
    "appropriateness = llema['appropriateness'].mean()\n",
    "appropriateness_int = st.t.interval(confidence=0.95, df=len(llema['appropriateness'].dropna())-1, loc=np.mean(llema['appropriateness'].dropna()), scale=st.sem(llema['appropriateness'].dropna()))\n",
    "appropriateness_sd = np.std(llema['appropriateness'])\n",
    "appropriateness_se = st.sem(llema['appropriateness'].dropna())\n",
    "topic = llema['topic'].mean()\n",
    "topic_int = st.t.interval(confidence=0.95, df=len(llema['topic'])-1, loc=np.mean(llema['topic']), scale=st.sem(llema['topic']))\n",
    "topic_sd = np.std(llema['topic'])\n",
    "topic_se = st.sem(llema['topic'])\n",
    "good = llema['good'].mean()\n",
    "good_int = st.t.interval(confidence=0.95, df=len(llema['good'])-1, loc=np.mean(llema['good']), scale=st.sem(llema['good']))\n",
    "good_sd = np.std(llema['good'])\n",
    "good_se = st.sem(llema['good'])\n",
    "execute = good * .2432288900690388\n",
    "execute_se = good_se * .2432288900690388\n",
    "print(f'''Llema metrics:\n",
    "Solvability: {solvability} Standard Deviation: {solvability_sd} Standard Error: {solvability_se*100}\n",
    "Accuracy: {accuracy} Standard Deviation: {accuracy_sd} Standard Error: {accuracy_se*100}\n",
    "Appropriateness: {appropriateness} Standard Deviation: {appropriateness_sd} Standard Error: {appropriateness_se*100}\n",
    "Percentage about Requested Topic: {topic} Standard Deviation: {topic_sd} Standard Error: {topic_se*100}\n",
    "Percentage Good in All Metrics: {good} Standard Deviation: {good_sd} Standard Error: {good_se*100}\n",
    "Percentage good with executable code: {execute} Standard Error: {execute_se*100}''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "b0bd798f-067c-454d-b78f-ccda5c58ab9e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mammoth metrics:\n",
      "Solvability: 0.868 Standard Deviation: 0.33849076796864047 Standard Error: 2.1450980824038166\n",
      "Accuracy: 0.9493087557603687 Standard Deviation: 0.21936645595229318 Standard Error: 1.4925996771273333\n",
      "Appropriateness: 0.6774193548387096 Standard Deviation: 0.46746376600611095 Standard Error: 3.180688055429901\n",
      "Percentage about Requested Topic: 0.976 Standard Deviation: 0.15304901175767194 Standard Error: 0.9699087026964266\n",
      "Percentage Good in All Metrics: 0.568 Standard Deviation: 0.49535441857320706 Standard Error: 3.139181076542942\n",
      "Percentage good with executable code: 0.039110049751243776 Standard Error: 0.21615057760872794\n"
     ]
    }
   ],
   "source": [
    "solvability = mammoth['solvability'].mean()\n",
    "solvability_int = st.t.interval(confidence=0.95, df=len(mammoth['solvability'])-1, loc=np.mean(mammoth['solvability']), scale=st.sem(mammoth['solvability']))\n",
    "solvability_sd = np.std(mammoth['solvability'])\n",
    "solvability_se = st.sem(mammoth['solvability'])\n",
    "accuracy = mammoth['accuracy'].mean()\n",
    "accuracy_int = st.t.interval(confidence=0.95, df=len(mammoth['accuracy'].dropna())-1, loc=np.mean(mammoth['accuracy'].dropna()), scale=st.sem(mammoth['accuracy'].dropna()))\n",
    "accuracy_sd = np.std(mammoth['accuracy'])\n",
    "accuracy_se = st.sem(mammoth['accuracy'].dropna())\n",
    "appropriateness = mammoth['appropriateness'].mean()\n",
    "appropriateness_int = st.t.interval(confidence=0.95, df=len(mammoth['appropriateness'].dropna())-1, loc=np.mean(mammoth['appropriateness'].dropna()), scale=st.sem(mammoth['appropriateness'].dropna()))\n",
    "appropriateness_sd = np.std(mammoth['appropriateness'])\n",
    "appropriateness_se = st.sem(mammoth['appropriateness'].dropna())\n",
    "topic = mammoth['topic'].mean()\n",
    "topic_int = st.t.interval(confidence=0.95, df=len(mammoth['topic'])-1, loc=np.mean(mammoth['topic']), scale=st.sem(mammoth['topic']))\n",
    "topic_sd = np.std(mammoth['topic'])\n",
    "topic_se = st.sem(mammoth['topic'])\n",
    "good = mammoth['good'].mean()\n",
    "good_int = st.t.interval(confidence=0.95, df=len(mammoth['good'])-1, loc=np.mean(mammoth['good']), scale=st.sem(mammoth['good']))\n",
    "good_sd = np.std(mammoth['good'])\n",
    "good_se = st.sem(mammoth['good'])\n",
    "execute = good * .06885572139303482\n",
    "execute_se = good_se * .06885572139303482\n",
    "print(f'''Mammoth metrics:\n",
    "Solvability: {solvability} Standard Deviation: {solvability_sd} Standard Error: {solvability_se*100}\n",
    "Accuracy: {accuracy} Standard Deviation: {accuracy_sd} Standard Error: {accuracy_se*100}\n",
    "Appropriateness: {appropriateness} Standard Deviation: {appropriateness_sd} Standard Error: {appropriateness_se*100}\n",
    "Percentage about Requested Topic: {topic} Standard Deviation: {topic_sd} Standard Error: {topic_se*100}\n",
    "Percentage Good in All Metrics: {good} Standard Deviation: {good_sd} Standard Error: {good_se*100}\n",
    "Percentage good with executable code: {execute} Standard Error: {execute_se*100}''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "1615d15c-7787-4791-8816-7c519f801077",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mathwell metrics:\n",
      "Solvability: 0.892 Standard Deviation: 0.3103804117530615 Standard Error: 1.9669559381568775\n",
      "Accuracy: 0.968609865470852 Standard Deviation: 0.17436970488989761 Standard Error: 1.1702928328273723\n",
      "Appropriateness: 0.8654708520179372 Standard Deviation: 0.34121995299994257 Standard Error: 2.2901183761575625\n",
      "Percentage about Requested Topic: 0.996 Standard Deviation: 0.063118935352238 Standard Error: 0.4000000000000001\n",
      "Percentage Good in All Metrics: 0.748 Standard Deviation: 0.43416126036301306 Standard Error: 2.7513851933031317\n",
      "Percentage good with executable code: 0.49631129104813326 Standard Error: 1.8255929645173732\n"
     ]
    }
   ],
   "source": [
    "solvability = mathwell['solvability'].mean()\n",
    "solvability_int = st.t.interval(confidence=0.95, df=len(mathwell['solvability'])-1, loc=np.mean(mathwell['solvability']), scale=st.sem(mathwell['solvability']))\n",
    "solvability_sd = np.std(mathwell['solvability'])\n",
    "solvability_se = st.sem(mathwell['solvability'])\n",
    "accuracy = mathwell['accuracy'].mean()\n",
    "accuracy_int = st.t.interval(confidence=0.95, df=len(mathwell['accuracy'].dropna())-1, loc=np.mean(mathwell['accuracy'].dropna()), scale=st.sem(mathwell['accuracy'].dropna()))\n",
    "accuracy_sd = np.std(mathwell['accuracy'])\n",
    "accuracy_se = st.sem(mathwell['accuracy'].dropna())\n",
    "appropriateness = mathwell['appropriateness'].mean()\n",
    "appropriateness_int = st.t.interval(confidence=0.95, df=len(mathwell['appropriateness'].dropna())-1, loc=np.mean(mathwell['appropriateness'].dropna()), scale=st.sem(mathwell['appropriateness'].dropna()))\n",
    "appropriateness_sd = np.std(mathwell['appropriateness'])\n",
    "appropriateness_se = st.sem(mathwell['appropriateness'].dropna())\n",
    "topic = mathwell['topic'].mean()\n",
    "topic_int = st.t.interval(confidence=0.95, df=len(mathwell['topic'])-1, loc=np.mean(mathwell['topic']), scale=st.sem(mathwell['topic']))\n",
    "topic_sd = np.std(mathwell['topic'])\n",
    "topic_se = st.sem(mathwell['topic'])\n",
    "good = mathwell['good'].mean()\n",
    "good_int = st.t.interval(confidence=0.95, df=len(mathwell['good'])-1, loc=np.mean(mathwell['good']), scale=st.sem(mathwell['good']))\n",
    "good_sd = np.std(mathwell['good'])\n",
    "good_se = st.sem(mathwell['good'])\n",
    "execute = good * .6635177687809268\n",
    "execute_se = good_se * .6635177687809268\n",
    "print(f'''Mathwell metrics:\n",
    "Solvability: {solvability} Standard Deviation: {solvability_sd} Standard Error: {solvability_se*100}\n",
    "Accuracy: {accuracy} Standard Deviation: {accuracy_sd} Standard Error: {accuracy_se*100}\n",
    "Appropriateness: {appropriateness} Standard Deviation: {appropriateness_sd} Standard Error: {appropriateness_se*100}\n",
    "Percentage about Requested Topic: {topic} Standard Deviation: {topic_sd} Standard Error: {topic_se*100}\n",
    "Percentage Good in All Metrics: {good} Standard Deviation: {good_sd} Standard Error: {good_se*100}\n",
    "Percentage good with executable code: {execute} Standard Error: {execute_se*100}''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "b5b1983c-37e4-49bd-abeb-df85830da73a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Llama_gsm metrics:\n",
      "Solvability: 0.74 Standard Deviation: 0.4386342439892262 Standard Error: 4.4084400227680804\n",
      "Accuracy: 0.7702702702702703 Standard Deviation: 0.4206589842236049 Standard Error: 4.923441009188971\n",
      "Appropriateness: 0.5405405405405406 Standard Deviation: 0.4983537544482642 Standard Error: 5.83278951301236\n",
      "Percentage about Requested Topic: 0.97 Standard Deviation: 0.17058722109231983 Standard Error: 1.7144660799776528\n",
      "Percentage Good in All Metrics: 0.35 Standard Deviation: 0.4769696007084728 Standard Error: 4.793724854411022\n",
      "Percentage good with executable code: 0.1416502946954813 Standard Error: 1.9400929666182132\n"
     ]
    }
   ],
   "source": [
    "solvability = llama_gsm['solvability'].mean()\n",
    "solvability_int = st.t.interval(confidence=0.95, df=len(llama_gsm['solvability'])-1, loc=np.mean(llama_gsm['solvability']), scale=st.sem(llama_gsm['solvability']))\n",
    "solvability_se = st.sem(llama_gsm['solvability'])\n",
    "solvability_sd = np.std(llama_gsm['solvability'])\n",
    "accuracy = llama_gsm['accuracy'].mean()\n",
    "accuracy_int = st.t.interval(confidence=0.95, df=len(llama_gsm['accuracy'].dropna())-1, loc=np.mean(llama_gsm['accuracy'].dropna()), scale=st.sem(llama_gsm['accuracy'].dropna()))\n",
    "accuracy_sd = np.std(llama_gsm['accuracy'])\n",
    "accuracy_se = st.sem(llama_gsm['accuracy'].dropna())\n",
    "appropriateness = llama_gsm['appropriateness'].mean()\n",
    "appropriateness_int = st.t.interval(confidence=0.95, df=len(llama_gsm['appropriateness'].dropna())-1, loc=np.mean(llama_gsm['appropriateness'].dropna()), scale=st.sem(llama_gsm['appropriateness'].dropna()))\n",
    "appropriateness_sd = np.std(llama_gsm['appropriateness'])\n",
    "appropriateness_se = st.sem(llama_gsm['appropriateness'].dropna())\n",
    "topic = llama_gsm['topic'].mean()\n",
    "topic_int = st.t.interval(confidence=0.95, df=len(llama_gsm['topic'])-1, loc=np.mean(llama_gsm['topic']), scale=st.sem(llama_gsm['topic']))\n",
    "topic_sd = np.std(llama_gsm['topic'])\n",
    "topic_se = st.sem(llama_gsm['topic'])\n",
    "good = llama_gsm['good'].mean()\n",
    "good_int = st.t.interval(confidence=0.95, df=len(llama_gsm['good'])-1, loc=np.mean(llama_gsm['good']), scale=st.sem(llama_gsm['good']))\n",
    "good_sd = np.std(llama_gsm['good'])\n",
    "good_se = st.sem(llama_gsm['good'])\n",
    "execute = good * .4047151277013752\n",
    "execute_se = good_se * .4047151277013752\n",
    "print(f'''Llama_gsm metrics:\n",
    "Solvability: {solvability} Standard Deviation: {solvability_sd} Standard Error: {solvability_se*100}\n",
    "Accuracy: {accuracy} Standard Deviation: {accuracy_sd} Standard Error: {accuracy_se*100}\n",
    "Appropriateness: {appropriateness} Standard Deviation: {appropriateness_sd} Standard Error: {appropriateness_se*100}\n",
    "Percentage about Requested Topic: {topic} Standard Deviation: {topic_sd} Standard Error: {topic_se*100}\n",
    "Percentage Good in All Metrics: {good} Standard Deviation: {good_sd} Standard Error: {good_se*100}\n",
    "Percentage good with executable code: {execute} Standard Error: {execute_se*100}''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "fe0b4b42-f883-4c66-b329-f16e78a1c605",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "egsm_only metrics:\n",
      "Solvability: 0.86 Standard Deviation: 0.3469870314579494 Standard Error: 2.1989409645240245\n",
      "Accuracy: 0.9162790697674419 Standard Deviation: 0.27696883592482624 Standard Error: 1.8933199846261417\n",
      "Appropriateness: 0.7674418604651163 Standard Deviation: 0.42246284010662677 Standard Error: 2.8878965218776047\n",
      "Percentage about Requested Topic: 0.984 Standard Deviation: 0.125475097130865 Standard Error: 0.7951661188874349\n",
      "Percentage Good in All Metrics: 0.608 Standard Deviation: 0.48819668167655544 Standard Error: 3.093820762040122\n",
      "Percentage good with executable code: 0.30957159221076747 Standard Error: 1.5752615449333724\n"
     ]
    }
   ],
   "source": [
    "solvability = egsm_only['solvability'].mean()\n",
    "solvability_int = st.t.interval(confidence=0.95, df=len(egsm_only['solvability'])-1, loc=np.mean(egsm_only['solvability']), scale=st.sem(egsm_only['solvability']))\n",
    "solvability_se = st.sem(egsm_only['solvability'])\n",
    "solvability_sd = np.std(egsm_only['solvability'])\n",
    "accuracy = egsm_only['accuracy'].mean()\n",
    "accuracy_int = st.t.interval(confidence=0.95, df=len(egsm_only['accuracy'].dropna())-1, loc=np.mean(egsm_only['accuracy'].dropna()), scale=st.sem(egsm_only['accuracy'].dropna()))\n",
    "accuracy_sd = np.std(egsm_only['accuracy'])\n",
    "accuracy_se = st.sem(egsm_only['accuracy'].dropna())\n",
    "appropriateness = egsm_only['appropriateness'].mean()\n",
    "appropriateness_int = st.t.interval(confidence=0.95, df=len(egsm_only['appropriateness'].dropna())-1, loc=np.mean(egsm_only['appropriateness'].dropna()), scale=st.sem(egsm_only['appropriateness'].dropna()))\n",
    "appropriateness_sd = np.std(egsm_only['appropriateness'])\n",
    "appropriateness_se = st.sem(egsm_only['appropriateness'].dropna())\n",
    "topic = egsm_only['topic'].mean()\n",
    "topic_int = st.t.interval(confidence=0.95, df=len(egsm_only['topic'])-1, loc=np.mean(egsm_only['topic']), scale=st.sem(egsm_only['topic']))\n",
    "topic_sd = np.std(egsm_only['topic'])\n",
    "topic_se = st.sem(egsm_only['topic'])\n",
    "good = egsm_only['good'].mean()\n",
    "good_int = st.t.interval(confidence=0.95, df=len(egsm_only['good'])-1, loc=np.mean(egsm_only['good']), scale=st.sem(egsm_only['good']))\n",
    "good_sd = np.std(egsm_only['good'])\n",
    "good_se = st.sem(egsm_only['good'])\n",
    "execute = good *  .5091638029782359\n",
    "execute_se = good_se * .5091638029782359\n",
    "print(f'''egsm_only metrics:\n",
    "Solvability: {solvability} Standard Deviation: {solvability_sd} Standard Error: {solvability_se*100}\n",
    "Accuracy: {accuracy} Standard Deviation: {accuracy_sd} Standard Error: {accuracy_se*100}\n",
    "Appropriateness: {appropriateness} Standard Deviation: {appropriateness_sd} Standard Error: {appropriateness_se*100}\n",
    "Percentage about Requested Topic: {topic} Standard Deviation: {topic_sd} Standard Error: {topic_se*100}\n",
    "Percentage Good in All Metrics: {good} Standard Deviation: {good_sd} Standard Error: {good_se*100}\n",
    "Percentage good with executable code: {execute} Standard Error: {execute_se*100}''')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02b5735b-c0ec-484a-a465-b8ce33c86113",
   "metadata": {},
   "source": [
    "### Overall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "2b1aac6e-4002-4e5c-ac63-6eb91e3f8dab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "gpt4 overall question statistics:\n",
      "Addition: 0.5316455696202531\n",
      "Subtraction: 0.4472573839662447\n",
      "Multiplication: 0.6160337552742616\n",
      "Division: 0.25316455696202533\n",
      "Fractions: 0.008438818565400843\n",
      "Decimals: 0.046413502109704644\n",
      "Total Operations: 1.9273504273504274\n",
      "No operations: 0.012658227848101266\n"
     ]
    }
   ],
   "source": [
    "addition = gpt4['addition'].mean()\n",
    "subtraction = gpt4['subtraction'].mean()\n",
    "multiplication = gpt4['multiplication'].mean()\n",
    "division = gpt4['division'].mean()\n",
    "fractions = gpt4['fractions'].mean()\n",
    "decimals = gpt4['decimals'].mean()\n",
    "total_ops = gpt4['total_ops'].mean()\n",
    "no_ops = gpt4['no_ops'].mean()\n",
    "print(len(gpt4[gpt4['no_ops']==1]))\n",
    "print(f\"gpt4 overall question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\\nNo operations: {no_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "1f14bbec-56c9-45a6-b8a5-c5ac1fc27516",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "gpt35 overall question statistics:\n",
      "Addition: 0.35294117647058826\n",
      "Subtraction: 0.2850678733031674\n",
      "Multiplication: 0.4434389140271493\n",
      "Division: 0.36199095022624433\n",
      "Fractions: 0.03167420814479638\n",
      "Decimals: 0.23076923076923078\n",
      "Total Operations: 1.737327188940092\n",
      "No operations: 0.01809954751131222\n"
     ]
    }
   ],
   "source": [
    "addition = gpt35['addition'].mean()\n",
    "subtraction = gpt35['subtraction'].mean()\n",
    "multiplication = gpt35['multiplication'].mean()\n",
    "division = gpt35['division'].mean()\n",
    "fractions = gpt35['fractions'].mean()\n",
    "decimals = gpt35['decimals'].mean()\n",
    "total_ops = gpt35['total_ops'].mean()\n",
    "no_ops = gpt35['no_ops'].mean()\n",
    "print(len(gpt35[gpt35['no_ops']==1]))\n",
    "print(f\"gpt35 overall question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\\nNo operations: {no_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "b477df46-8967-4530-9376-294d76104b26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "Llama overall question statistics:\n",
      "Addition: 0.5761904761904761\n",
      "Subtraction: 0.5857142857142857\n",
      "Multiplication: 0.22857142857142856\n",
      "Division: 0.14285714285714285\n",
      "Fractions: 0.08095238095238096\n",
      "Decimals: 0.11428571428571428\n",
      "Total Operations: 1.815\n",
      "No operations: 0.047619047619047616\n"
     ]
    }
   ],
   "source": [
    "addition = llama['addition'].mean()\n",
    "subtraction = llama['subtraction'].mean()\n",
    "multiplication = llama['multiplication'].mean()\n",
    "division = llama['division'].mean()\n",
    "fractions = llama['fractions'].mean()\n",
    "decimals = llama['decimals'].mean()\n",
    "total_ops = llama['total_ops'].mean()\n",
    "no_ops = llama['no_ops'].mean()\n",
    "print(len(llama[llama['no_ops']==1]))\n",
    "print(f\"Llama overall question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\\nNo operations: {no_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "b5525818-2339-4bf9-9cc7-04a9b7c2df7f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19\n",
      "Llema question statistics:\n",
      "Addition: 0.3442622950819672\n",
      "Subtraction: 0.27049180327868855\n",
      "Multiplication: 0.3360655737704918\n",
      "Division: 0.20491803278688525\n",
      "Fractions: 0.06557377049180328\n",
      "Decimals: 0.1557377049180328\n",
      "Total Operations: 1.6310679611650485\n",
      "No operations: 0.1557377049180328\n"
     ]
    }
   ],
   "source": [
    "addition = llema['addition'].mean()\n",
    "subtraction = llema['subtraction'].mean()\n",
    "multiplication = llema['multiplication'].mean()\n",
    "division = llema['division'].mean()\n",
    "fractions = llema['fractions'].mean()\n",
    "decimals = llema['decimals'].mean()\n",
    "total_ops = llema['total_ops'].mean()\n",
    "no_ops = llema['no_ops'].mean()\n",
    "print(len(llema[llema['no_ops']==1]))\n",
    "print(f\"Llema question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\\nNo operations: {no_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "ab1b5d42-47ee-4f39-a989-87a0b2e0798a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n",
      "Mammoth question statistics:\n",
      "Addition: 0.39631336405529954\n",
      "Subtraction: 0.3778801843317972\n",
      "Multiplication: 0.4377880184331797\n",
      "Division: 0.1935483870967742\n",
      "Fractions: 0.03686635944700461\n",
      "Decimals: 0.10599078341013825\n",
      "Total Operations: 1.5849056603773586\n",
      "No operations: 0.02304147465437788\n"
     ]
    }
   ],
   "source": [
    "addition = mammoth['addition'].mean()\n",
    "subtraction = mammoth['subtraction'].mean()\n",
    "multiplication = mammoth['multiplication'].mean()\n",
    "division = mammoth['division'].mean()\n",
    "fractions = mammoth['fractions'].mean()\n",
    "decimals = mammoth['decimals'].mean()\n",
    "total_ops = mammoth['total_ops'].mean()\n",
    "no_ops = mammoth['no_ops'].mean()\n",
    "print(len(mammoth[mammoth['no_ops']==1]))\n",
    "print(f\"Mammoth question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\\nNo operations: {no_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "86280641-03b4-414a-affc-b6b36f066cfe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "Mathwell question statistics:\n",
      "Addition: 0.695067264573991\n",
      "Subtraction: 0.6905829596412556\n",
      "Multiplication: 0.24663677130044842\n",
      "Division: 0.1031390134529148\n",
      "Fractions: 0.053811659192825115\n",
      "Decimals: 0.07623318385650224\n",
      "Total Operations: 1.8909090909090909\n",
      "No operations: 0.013452914798206279\n"
     ]
    }
   ],
   "source": [
    "addition = mathwell['addition'].mean()\n",
    "subtraction = mathwell['subtraction'].mean()\n",
    "multiplication = mathwell['multiplication'].mean()\n",
    "division = mathwell['division'].mean()\n",
    "fractions = mathwell['fractions'].mean()\n",
    "decimals = mathwell['decimals'].mean()\n",
    "total_ops = mathwell['total_ops'].mean()\n",
    "no_ops = mathwell['no_ops'].mean()\n",
    "print(len(mathwell[mathwell['no_ops']==1]))\n",
    "print(f\"Mathwell question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\\nNo operations: {no_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "81019edc-13ce-4a2b-8ad5-7374e2709a21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n",
      "Llama_gsm overall question statistics:\n",
      "Addition: 0.28378378378378377\n",
      "Subtraction: 0.25675675675675674\n",
      "Multiplication: 0.3783783783783784\n",
      "Division: 0.32432432432432434\n",
      "Fractions: 0.05405405405405406\n",
      "Decimals: 0.2972972972972973\n",
      "Total Operations: 1.7352941176470589\n",
      "No operations: 0.08108108108108109\n"
     ]
    }
   ],
   "source": [
    "addition = llama_gsm['addition'].mean()\n",
    "subtraction = llama_gsm['subtraction'].mean()\n",
    "multiplication = llama_gsm['multiplication'].mean()\n",
    "division = llama_gsm['division'].mean()\n",
    "fractions = llama_gsm['fractions'].mean()\n",
    "decimals = llama_gsm['decimals'].mean()\n",
    "total_ops = llama_gsm['total_ops'].mean()\n",
    "no_ops = llama_gsm['no_ops'].mean()\n",
    "print(len(llama_gsm[llama_gsm['no_ops']==1]))\n",
    "print(f\"Llama_gsm overall question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\\nNo operations: {no_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "7d06cd8a-8ed4-410b-a1ae-6fe9dfc259c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "EGSM Only overall question statistics:\n",
      "Addition: 0.6372093023255814\n",
      "Subtraction: 0.6744186046511628\n",
      "Multiplication: 0.20465116279069767\n",
      "Division: 0.10697674418604651\n",
      "Fractions: 0.04186046511627907\n",
      "Decimals: 0.10697674418604651\n",
      "Total Operations: 1.8405797101449275\n",
      "No operations: 0.037209302325581395\n"
     ]
    }
   ],
   "source": [
    "addition = egsm_only['addition'].mean()\n",
    "subtraction = egsm_only['subtraction'].mean()\n",
    "multiplication = egsm_only['multiplication'].mean()\n",
    "division = egsm_only['division'].mean()\n",
    "fractions = egsm_only['fractions'].mean()\n",
    "decimals = egsm_only['decimals'].mean()\n",
    "total_ops = egsm_only['total_ops'].mean()\n",
    "no_ops = egsm_only['no_ops'].mean()\n",
    "print(len(egsm_only[egsm_only['no_ops']==1]))\n",
    "print(f\"EGSM Only overall question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\\nNo operations: {no_ops}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee525d48-8e49-4db3-a965-8cbdadfea0ae",
   "metadata": {},
   "source": [
    "### Good Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "0850ab9f-1d05-45eb-8cd3-585c32c5714c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpt4 good question statistics:\n",
      "Addition: 0.5431472081218274\n",
      "Subtraction: 0.4517766497461929\n",
      "Multiplication: 0.5989847715736041\n",
      "Division: 0.25380710659898476\n",
      "Fractions: 0.01015228426395939\n",
      "Decimals: 0.030456852791878174\n",
      "Total Operations: 1.8883248730964468\n"
     ]
    }
   ],
   "source": [
    "good_gpt4 = gpt4.query('good==1')\n",
    "addition = good_gpt4['addition'].mean()\n",
    "subtraction = good_gpt4['subtraction'].mean()\n",
    "multiplication = good_gpt4['multiplication'].mean()\n",
    "division = good_gpt4['division'].mean()\n",
    "fractions = good_gpt4['fractions'].mean()\n",
    "decimals = good_gpt4['decimals'].mean()\n",
    "total_ops = good_gpt4['total_ops'].mean()\n",
    "print(f\"gpt4 good question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "5c43577e-3227-4b54-b2fd-d2616e6f33fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpt35 good question statistics:\n",
      "Addition: 0.3630573248407643\n",
      "Subtraction: 0.29936305732484075\n",
      "Multiplication: 0.39490445859872614\n",
      "Division: 0.35668789808917195\n",
      "Fractions: 0.03184713375796178\n",
      "Decimals: 0.21019108280254778\n",
      "Total Operations: 1.6560509554140128\n"
     ]
    }
   ],
   "source": [
    "good_gpt35 = gpt35.query('good==1')\n",
    "addition = good_gpt35['addition'].mean()\n",
    "subtraction = good_gpt35['subtraction'].mean()\n",
    "multiplication = good_gpt35['multiplication'].mean()\n",
    "division = good_gpt35['division'].mean()\n",
    "fractions = good_gpt35['fractions'].mean()\n",
    "decimals = good_gpt35['decimals'].mean()\n",
    "total_ops = good_gpt35['total_ops'].mean()\n",
    "print(f\"gpt35 good question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "0147a9b4-f3a3-40cc-9303-54208ed5a6cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Llama good question statistics:\n",
      "Addition: 0.5961538461538461\n",
      "Subtraction: 0.6025641025641025\n",
      "Multiplication: 0.24358974358974358\n",
      "Division: 0.1282051282051282\n",
      "Fractions: 0.057692307692307696\n",
      "Decimals: 0.08974358974358974\n",
      "Total Operations: 1.7179487179487178\n"
     ]
    }
   ],
   "source": [
    "good_llama = llama.query('good==1')\n",
    "addition = good_llama['addition'].mean()\n",
    "subtraction = good_llama['subtraction'].mean()\n",
    "multiplication = good_llama['multiplication'].mean()\n",
    "division = good_llama['division'].mean()\n",
    "fractions = good_llama['fractions'].mean()\n",
    "decimals = good_llama['decimals'].mean()\n",
    "total_ops = good_llama['total_ops'].mean()\n",
    "print(f\"Llama good question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "db12633a-dfc4-41d9-accf-2f10c33e9dea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Llema good question statistics:\n",
      "Addition: 0.3684210526315789\n",
      "Subtraction: 0.39473684210526316\n",
      "Multiplication: 0.3157894736842105\n",
      "Division: 0.15789473684210525\n",
      "Fractions: 0.02631578947368421\n",
      "Decimals: 0.13157894736842105\n",
      "Total Operations: 1.394736842105263\n"
     ]
    }
   ],
   "source": [
    "good_llema = llema.query('good==1')\n",
    "addition = good_llema['addition'].mean()\n",
    "subtraction = good_llema['subtraction'].mean()\n",
    "multiplication = good_llema['multiplication'].mean()\n",
    "division = good_llema['division'].mean()\n",
    "fractions = good_llema['fractions'].mean()\n",
    "decimals = good_llema['decimals'].mean()\n",
    "total_ops = good_llema['total_ops'].mean()\n",
    "print(f\"Llema good question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "e8a9f27a-2e8e-462c-93ab-24c2211cd1ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mammoth good question statistics:\n",
      "Addition: 0.4295774647887324\n",
      "Subtraction: 0.4225352112676056\n",
      "Multiplication: 0.4084507042253521\n",
      "Division: 0.16901408450704225\n",
      "Fractions: 0.04929577464788732\n",
      "Decimals: 0.09859154929577464\n",
      "Total Operations: 1.5774647887323943\n"
     ]
    }
   ],
   "source": [
    "good_mammoth = mammoth.query('good==1')\n",
    "addition = good_mammoth['addition'].mean()\n",
    "subtraction = good_mammoth['subtraction'].mean()\n",
    "multiplication = good_mammoth['multiplication'].mean()\n",
    "division = good_mammoth['division'].mean()\n",
    "fractions = good_mammoth['fractions'].mean()\n",
    "decimals = good_mammoth['decimals'].mean()\n",
    "total_ops = good_mammoth['total_ops'].mean()\n",
    "print(f\"Mammoth good question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "afd6842d-fd04-4156-911a-47d9a928fcf3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mathwell good question statistics:\n",
      "Addition: 0.7112299465240641\n",
      "Subtraction: 0.7058823529411765\n",
      "Multiplication: 0.24598930481283424\n",
      "Division: 0.0855614973262032\n",
      "Fractions: 0.0481283422459893\n",
      "Decimals: 0.0748663101604278\n",
      "Total Operations: 1.8716577540106951\n"
     ]
    }
   ],
   "source": [
    "good_mathwell = mathwell.query('good==1')\n",
    "addition = good_mathwell['addition'].mean()\n",
    "subtraction = good_mathwell['subtraction'].mean()\n",
    "multiplication = good_mathwell['multiplication'].mean()\n",
    "division = good_mathwell['division'].mean()\n",
    "fractions = good_mathwell['fractions'].mean()\n",
    "decimals = good_mathwell['decimals'].mean()\n",
    "total_ops = good_mathwell['total_ops'].mean()\n",
    "print(f\"Mathwell good question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "7a518fa0-9c93-4d24-aa67-db67dc51d520",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Llama_gsm good question statistics:\n",
      "Addition: 0.37142857142857144\n",
      "Subtraction: 0.2571428571428571\n",
      "Multiplication: 0.34285714285714286\n",
      "Division: 0.3142857142857143\n",
      "Fractions: 0.05714285714285714\n",
      "Decimals: 0.4\n",
      "Total Operations: 1.7428571428571429\n"
     ]
    }
   ],
   "source": [
    "good_llama_gsm = llama_gsm.query('good==1')\n",
    "addition = good_llama_gsm['addition'].mean()\n",
    "subtraction = good_llama_gsm['subtraction'].mean()\n",
    "multiplication = good_llama_gsm['multiplication'].mean()\n",
    "division = good_llama_gsm['division'].mean()\n",
    "fractions = good_llama_gsm['fractions'].mean()\n",
    "decimals = good_llama_gsm['decimals'].mean()\n",
    "total_ops = good_llama_gsm['total_ops'].mean()\n",
    "print(f\"Llama_gsm good question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "ce327526-48e3-453e-8bc0-acd5bdba66dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "egsm_only good question statistics:\n",
      "Addition: 0.7302631578947368\n",
      "Subtraction: 0.743421052631579\n",
      "Multiplication: 0.14473684210526316\n",
      "Division: 0.07236842105263158\n",
      "Fractions: 0.019736842105263157\n",
      "Decimals: 0.05263157894736842\n",
      "Total Operations: 1.763157894736842\n"
     ]
    }
   ],
   "source": [
    "good_egsm_only = egsm_only.query('good==1')\n",
    "addition = good_egsm_only['addition'].mean()\n",
    "subtraction = good_egsm_only['subtraction'].mean()\n",
    "multiplication = good_egsm_only['multiplication'].mean()\n",
    "division = good_egsm_only['division'].mean()\n",
    "fractions = good_egsm_only['fractions'].mean()\n",
    "decimals = good_egsm_only['decimals'].mean()\n",
    "total_ops = good_egsm_only['total_ops'].mean()\n",
    "print(f\"egsm_only good question statistics:\\nAddition: {addition}\\nSubtraction: {subtraction}\\nMultiplication: {multiplication}\\nDivision: {division}\\nFractions: {fractions}\\nDecimals: {decimals}\\nTotal Operations: {total_ops}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "533a0020-7a52-45d4-b37a-dee3b88c746a",
   "metadata": {},
   "source": [
    "## Check for Statistical Significance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "5e46b80c-cc68-4079-b50c-7b13b163b5c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: -1.0585420185859666\n",
      "P-value: 0.2903215706618074\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "# Perform an independent two-sample t-test on percentage good\n",
    "t_statistic, p_value = stats.ttest_ind(mathwell['good'], gpt4['good'])\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "a7b97017-1cb7-4e3c-bfb2-d3dfbb08712a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 3.008081382179526\n",
      "P-value: 0.0027620491234694095\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "# Perform an independent two-sample t-test on percentage good\n",
    "t_statistic, p_value = stats.ttest_ind(mathwell['good'], llama['good'])\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "ff44ebc4-b1de-4c3f-ae78-c22d702043f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 2.914520203927901\n",
      "P-value: 0.0037226966148345628\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "# Perform an independent two-sample t-test on percentage good\n",
    "t_statistic, p_value = stats.ttest_ind(mathwell['good'], gpt35['good'])\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "89f13d19-5b53-407f-8cbd-9067976d2288",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: -2.3155773543221763\n",
      "P-value: 0.020986760690813174\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "# Perform an independent two-sample t-test on solvability\n",
    "t_statistic, p_value = stats.ttest_ind(mathwell['solvability'], gpt4['solvability'])\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "149257f9-511d-4c06-8e12-1f918d611497",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 0.8246321644499606\n",
      "P-value: 0.4099752507629809\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "# Perform an independent two-sample t-test on solvability\n",
    "t_statistic, p_value = stats.ttest_ind(mathwell['solvability'], mammoth['solvability'])\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "cdb693c6-7512-477e-9639-0fe10110ff84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 1.0207634545824231\n",
      "P-value: 0.3079299894123785\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "# Perform an independent two-sample t-test on accuracy\n",
    "mathwell_clean = mathwell['accuracy'].dropna()\n",
    "mammoth_clean = mammoth['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(mathwell_clean, mammoth_clean)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "f4bbd8a0-349b-41d1-9012-a6fbb29a3c96",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 1.5813637792343462\n",
      "P-value: 0.1145284167264593\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "# Perform an independent two-sample t-test on appropriateness\n",
    "mathwell_clean = mathwell['appropriateness'].dropna()\n",
    "llama_clean = llama['appropriateness'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(mathwell_clean, llama_clean)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "39cf2f27-5b15-42e2-adb0-cd92d41f1da6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 0.9976153121103332\n",
      "P-value: 0.31904308346725163\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "# Perform an independent two-sample t-test on number of operations\n",
    "mathwell_clean = mathwell['total_ops'].dropna()\n",
    "llama_clean = llama['total_ops'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(mathwell_clean, llama_clean)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "986fcd79-0600-4a77-9be9-941ef12ebea7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: -0.7513903197323029\n",
      "P-value: 0.45282131522853575\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "\n",
    "# Perform an independent two-sample t-test on number of no ops\n",
    "mathwell_clean = mathwell['no_ops'].dropna()\n",
    "mammoth_clean = mammoth['no_ops'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(mathwell_clean, mammoth_clean)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "9924e0b8-b39e-4be5-8fa5-e894c0e3deeb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 6.029028726239575\n",
      "P-value: 3.215402036842197e-09\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on percentage good\n",
    "llama_execute =llama['good'] * .55425904317386234\n",
    "mathwell_execute = mathwell['good'] * .6635177687809268\n",
    "t_statistic, p_value = stats.ttest_ind(mathwell_execute, llama_execute)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "4b82caf8-340c-4560-b33d-db6376413661",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 2.507153982425123\n",
      "P-value: 0.012488098413798027\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on percentage good\n",
    "gpt35_execute =gpt35['good'] * .9754269054560599\n",
    "gpt4_execute = gpt4['good'] * .6675392670157068\n",
    "t_statistic, p_value = stats.ttest_ind(gpt35_execute, gpt4_execute)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5052c999-9144-48b5-b43c-1efdaafc4d07",
   "metadata": {},
   "source": [
    "## Build Logit Models to Control for Question Type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "fc82c19b-9c5a-406e-9da3-91b88872d72f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.575103\n",
      "         Iterations 5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>Logit Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>         <td>good</td>       <th>  No. Observations:  </th>  <td>  1518</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                 <td>Logit</td>      <th>  Df Residuals:      </th>  <td>  1512</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>                 <td>MLE</td>       <th>  Df Model:          </th>  <td>     5</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>            <td>Thu, 27 Jun 2024</td> <th>  Pseudo R-squ.:     </th>  <td>0.05733</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                <td>14:50:51</td>     <th>  Log-Likelihood:    </th> <td> -873.01</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>converged:</th>             <td>True</td>       <th>  LL-Null:           </th> <td> -926.10</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>     <td>nonrobust</td>    <th>  LLR p-value:       </th> <td>2.609e-21</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "     <td></td>        <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>   <td>    0.9970</td> <td>    0.100</td> <td>   10.010</td> <td> 0.000</td> <td>    0.802</td> <td>    1.192</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>llema</th>   <td>   -1.7902</td> <td>    0.219</td> <td>   -8.159</td> <td> 0.000</td> <td>   -2.220</td> <td>   -1.360</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>mammoth</th> <td>   -0.3587</td> <td>    0.174</td> <td>   -2.061</td> <td> 0.039</td> <td>   -0.700</td> <td>   -0.018</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>llama</th>   <td>    0.0639</td> <td>    0.187</td> <td>    0.342</td> <td> 0.732</td> <td>   -0.302</td> <td>    0.430</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gpt35</th>   <td>   -0.0839</td> <td>    0.179</td> <td>   -0.468</td> <td> 0.640</td> <td>   -0.435</td> <td>    0.268</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gpt4</th>    <td>    0.5973</td> <td>    0.200</td> <td>    2.987</td> <td> 0.003</td> <td>    0.205</td> <td>    0.989</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/latex": [
       "\\begin{center}\n",
       "\\begin{tabular}{lclc}\n",
       "\\toprule\n",
       "\\textbf{Dep. Variable:}   &       good       & \\textbf{  No. Observations:  } &     1518    \\\\\n",
       "\\textbf{Model:}           &      Logit       & \\textbf{  Df Residuals:      } &     1512    \\\\\n",
       "\\textbf{Method:}          &       MLE        & \\textbf{  Df Model:          } &        5    \\\\\n",
       "\\textbf{Date:}            & Thu, 27 Jun 2024 & \\textbf{  Pseudo R-squ.:     } &  0.05733    \\\\\n",
       "\\textbf{Time:}            &     14:50:51     & \\textbf{  Log-Likelihood:    } &   -873.01   \\\\\n",
       "\\textbf{converged:}       &       True       & \\textbf{  LL-Null:           } &   -926.10   \\\\\n",
       "\\textbf{Covariance Type:} &    nonrobust     & \\textbf{  LLR p-value:       } & 2.609e-21   \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lcccccc}\n",
       "                 & \\textbf{coef} & \\textbf{std err} & \\textbf{z} & \\textbf{P$> |$z$|$} & \\textbf{[0.025} & \\textbf{0.975]}  \\\\\n",
       "\\midrule\n",
       "\\textbf{const}   &       0.9970  &        0.100     &    10.010  &         0.000        &        0.802    &        1.192     \\\\\n",
       "\\textbf{llema}   &      -1.7902  &        0.219     &    -8.159  &         0.000        &       -2.220    &       -1.360     \\\\\n",
       "\\textbf{mammoth} &      -0.3587  &        0.174     &    -2.061  &         0.039        &       -0.700    &       -0.018     \\\\\n",
       "\\textbf{llama}   &       0.0639  &        0.187     &     0.342  &         0.732        &       -0.302    &        0.430     \\\\\n",
       "\\textbf{gpt35}   &      -0.0839  &        0.179     &    -0.468  &         0.640        &       -0.435    &        0.268     \\\\\n",
       "\\textbf{gpt4}    &       0.5973  &        0.200     &     2.987  &         0.003        &        0.205    &        0.989     \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "%\\caption{Logit Regression Results}\n",
       "\\end{center}"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                           Logit Regression Results                           \n",
       "==============================================================================\n",
       "Dep. Variable:                   good   No. Observations:                 1518\n",
       "Model:                          Logit   Df Residuals:                     1512\n",
       "Method:                           MLE   Df Model:                            5\n",
       "Date:                Thu, 27 Jun 2024   Pseudo R-squ.:                 0.05733\n",
       "Time:                        14:50:51   Log-Likelihood:                -873.01\n",
       "converged:                       True   LL-Null:                       -926.10\n",
       "Covariance Type:            nonrobust   LLR p-value:                 2.609e-21\n",
       "==============================================================================\n",
       "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "const          0.9970      0.100     10.010      0.000       0.802       1.192\n",
       "llema         -1.7902      0.219     -8.159      0.000      -2.220      -1.360\n",
       "mammoth       -0.3587      0.174     -2.061      0.039      -0.700      -0.018\n",
       "llama          0.0639      0.187      0.342      0.732      -0.302       0.430\n",
       "gpt35         -0.0839      0.179     -0.468      0.640      -0.435       0.268\n",
       "gpt4           0.5973      0.200      2.987      0.003       0.205       0.989\n",
       "==============================================================================\n",
       "\"\"\""
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_df = df[df['model']!=\"llama_gsm\"]\n",
    "reg_df = pd.concat([df, pd.get_dummies(df['model'], dtype = int)], axis=1)\n",
    "#reg_df.to_csv('data/evaluation_annotations.csv', index=False)\n",
    "reg_df = reg_df[reg_df['solvability']==1]\n",
    "x = reg_df[['llema', 'mammoth', 'llama', 'gpt35', 'gpt4']]\n",
    "x = sm.add_constant(x)\n",
    "y = reg_df['good']\n",
    "log_reg = sm.Logit(y, x).fit() \n",
    "log_reg.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "db461348-6e77-438b-83f7-b36356a118b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.561031\n",
      "         Iterations 6\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>Logit Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>         <td>good</td>       <th>  No. Observations:  </th>  <td>  1518</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                 <td>Logit</td>      <th>  Df Residuals:      </th>  <td>  1506</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>                 <td>MLE</td>       <th>  Df Model:          </th>  <td>    11</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>            <td>Thu, 27 Jun 2024</td> <th>  Pseudo R-squ.:     </th>  <td>0.08040</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                <td>14:50:52</td>     <th>  Log-Likelihood:    </th> <td> -851.64</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>converged:</th>             <td>True</td>       <th>  LL-Null:           </th> <td> -926.10</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>     <td>nonrobust</td>    <th>  LLR p-value:       </th> <td>2.481e-26</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "         <td></td>           <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>          <td>    0.6982</td> <td>    0.174</td> <td>    4.002</td> <td> 0.000</td> <td>    0.356</td> <td>    1.040</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>addition</th>       <td>    0.3832</td> <td>    0.134</td> <td>    2.867</td> <td> 0.004</td> <td>    0.121</td> <td>    0.645</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>subtraction</th>    <td>    0.3213</td> <td>    0.146</td> <td>    2.196</td> <td> 0.028</td> <td>    0.034</td> <td>    0.608</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>multiplication</th> <td>   -0.1256</td> <td>    0.148</td> <td>   -0.850</td> <td> 0.395</td> <td>   -0.415</td> <td>    0.164</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>division</th>       <td>   -0.1009</td> <td>    0.172</td> <td>   -0.587</td> <td> 0.557</td> <td>   -0.438</td> <td>    0.236</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>fractions</th>      <td>   -0.2087</td> <td>    0.282</td> <td>   -0.741</td> <td> 0.458</td> <td>   -0.761</td> <td>    0.343</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>decimals</th>       <td>   -0.3706</td> <td>    0.183</td> <td>   -2.027</td> <td> 0.043</td> <td>   -0.729</td> <td>   -0.012</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>llema</th>          <td>   -1.6010</td> <td>    0.226</td> <td>   -7.084</td> <td> 0.000</td> <td>   -2.044</td> <td>   -1.158</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>mammoth</th>        <td>   -0.1888</td> <td>    0.181</td> <td>   -1.044</td> <td> 0.296</td> <td>   -0.543</td> <td>    0.166</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>llama</th>          <td>    0.0925</td> <td>    0.190</td> <td>    0.486</td> <td> 0.627</td> <td>   -0.281</td> <td>    0.466</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gpt35</th>          <td>    0.2021</td> <td>    0.189</td> <td>    1.069</td> <td> 0.285</td> <td>   -0.168</td> <td>    0.573</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gpt4</th>           <td>    0.7025</td> <td>    0.212</td> <td>    3.321</td> <td> 0.001</td> <td>    0.288</td> <td>    1.117</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/latex": [
       "\\begin{center}\n",
       "\\begin{tabular}{lclc}\n",
       "\\toprule\n",
       "\\textbf{Dep. Variable:}   &       good       & \\textbf{  No. Observations:  } &     1518    \\\\\n",
       "\\textbf{Model:}           &      Logit       & \\textbf{  Df Residuals:      } &     1506    \\\\\n",
       "\\textbf{Method:}          &       MLE        & \\textbf{  Df Model:          } &       11    \\\\\n",
       "\\textbf{Date:}            & Thu, 27 Jun 2024 & \\textbf{  Pseudo R-squ.:     } &  0.08040    \\\\\n",
       "\\textbf{Time:}            &     14:50:52     & \\textbf{  Log-Likelihood:    } &   -851.64   \\\\\n",
       "\\textbf{converged:}       &       True       & \\textbf{  LL-Null:           } &   -926.10   \\\\\n",
       "\\textbf{Covariance Type:} &    nonrobust     & \\textbf{  LLR p-value:       } & 2.481e-26   \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lcccccc}\n",
       "                        & \\textbf{coef} & \\textbf{std err} & \\textbf{z} & \\textbf{P$> |$z$|$} & \\textbf{[0.025} & \\textbf{0.975]}  \\\\\n",
       "\\midrule\n",
       "\\textbf{const}          &       0.6982  &        0.174     &     4.002  &         0.000        &        0.356    &        1.040     \\\\\n",
       "\\textbf{addition}       &       0.3832  &        0.134     &     2.867  &         0.004        &        0.121    &        0.645     \\\\\n",
       "\\textbf{subtraction}    &       0.3213  &        0.146     &     2.196  &         0.028        &        0.034    &        0.608     \\\\\n",
       "\\textbf{multiplication} &      -0.1256  &        0.148     &    -0.850  &         0.395        &       -0.415    &        0.164     \\\\\n",
       "\\textbf{division}       &      -0.1009  &        0.172     &    -0.587  &         0.557        &       -0.438    &        0.236     \\\\\n",
       "\\textbf{fractions}      &      -0.2087  &        0.282     &    -0.741  &         0.458        &       -0.761    &        0.343     \\\\\n",
       "\\textbf{decimals}       &      -0.3706  &        0.183     &    -2.027  &         0.043        &       -0.729    &       -0.012     \\\\\n",
       "\\textbf{llema}          &      -1.6010  &        0.226     &    -7.084  &         0.000        &       -2.044    &       -1.158     \\\\\n",
       "\\textbf{mammoth}        &      -0.1888  &        0.181     &    -1.044  &         0.296        &       -0.543    &        0.166     \\\\\n",
       "\\textbf{llama}          &       0.0925  &        0.190     &     0.486  &         0.627        &       -0.281    &        0.466     \\\\\n",
       "\\textbf{gpt35}          &       0.2021  &        0.189     &     1.069  &         0.285        &       -0.168    &        0.573     \\\\\n",
       "\\textbf{gpt4}           &       0.7025  &        0.212     &     3.321  &         0.001        &        0.288    &        1.117     \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "%\\caption{Logit Regression Results}\n",
       "\\end{center}"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                           Logit Regression Results                           \n",
       "==============================================================================\n",
       "Dep. Variable:                   good   No. Observations:                 1518\n",
       "Model:                          Logit   Df Residuals:                     1506\n",
       "Method:                           MLE   Df Model:                           11\n",
       "Date:                Thu, 27 Jun 2024   Pseudo R-squ.:                 0.08040\n",
       "Time:                        14:50:52   Log-Likelihood:                -851.64\n",
       "converged:                       True   LL-Null:                       -926.10\n",
       "Covariance Type:            nonrobust   LLR p-value:                 2.481e-26\n",
       "==================================================================================\n",
       "                     coef    std err          z      P>|z|      [0.025      0.975]\n",
       "----------------------------------------------------------------------------------\n",
       "const              0.6982      0.174      4.002      0.000       0.356       1.040\n",
       "addition           0.3832      0.134      2.867      0.004       0.121       0.645\n",
       "subtraction        0.3213      0.146      2.196      0.028       0.034       0.608\n",
       "multiplication    -0.1256      0.148     -0.850      0.395      -0.415       0.164\n",
       "division          -0.1009      0.172     -0.587      0.557      -0.438       0.236\n",
       "fractions         -0.2087      0.282     -0.741      0.458      -0.761       0.343\n",
       "decimals          -0.3706      0.183     -2.027      0.043      -0.729      -0.012\n",
       "llema             -1.6010      0.226     -7.084      0.000      -2.044      -1.158\n",
       "mammoth           -0.1888      0.181     -1.044      0.296      -0.543       0.166\n",
       "llama              0.0925      0.190      0.486      0.627      -0.281       0.466\n",
       "gpt35              0.2021      0.189      1.069      0.285      -0.168       0.573\n",
       "gpt4               0.7025      0.212      3.321      0.001       0.288       1.117\n",
       "==================================================================================\n",
       "\"\"\""
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = reg_df[['addition', 'subtraction',\n",
    "       'multiplication', 'division', 'fractions', 'decimals', \n",
    "       'llema', 'mammoth', 'llama', 'gpt35', 'gpt4']]\n",
    "y = reg_df['good']\n",
    "x = sm.add_constant(x)\n",
    "log_reg = sm.Logit(y, x).fit() \n",
    "log_reg.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "0c866b54-150b-4454-9fc1-8bd56198996a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.299148\n",
      "         Iterations 7\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>Logit Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>       <td>accuracy</td>     <th>  No. Observations:  </th>  <td>  1518</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                 <td>Logit</td>      <th>  Df Residuals:      </th>  <td>  1512</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>                 <td>MLE</td>       <th>  Df Model:          </th>  <td>     5</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>            <td>Thu, 27 Jun 2024</td> <th>  Pseudo R-squ.:     </th>  <td>0.08060</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                <td>14:50:52</td>     <th>  Log-Likelihood:    </th> <td> -454.11</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>converged:</th>             <td>True</td>       <th>  LL-Null:           </th> <td> -493.92</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>     <td>nonrobust</td>    <th>  LLR p-value:       </th> <td>1.009e-15</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "     <td></td>        <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>   <td>    2.4151</td> <td>    0.161</td> <td>   14.996</td> <td> 0.000</td> <td>    2.099</td> <td>    2.731</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>llema</th>   <td>   -1.8425</td> <td>    0.248</td> <td>   -7.431</td> <td> 0.000</td> <td>   -2.329</td> <td>   -1.357</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>llama</th>   <td>   -0.2697</td> <td>    0.277</td> <td>   -0.974</td> <td> 0.330</td> <td>   -0.813</td> <td>    0.273</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>mammoth</th> <td>    0.5149</td> <td>    0.349</td> <td>    1.476</td> <td> 0.140</td> <td>   -0.169</td> <td>    1.199</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gpt35</th>   <td>   -0.2674</td> <td>    0.273</td> <td>   -0.980</td> <td> 0.327</td> <td>   -0.802</td> <td>    0.268</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gpt4</th>    <td>    0.7073</td> <td>    0.361</td> <td>    1.959</td> <td> 0.050</td> <td>   -0.000</td> <td>    1.415</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/latex": [
       "\\begin{center}\n",
       "\\begin{tabular}{lclc}\n",
       "\\toprule\n",
       "\\textbf{Dep. Variable:}   &     accuracy     & \\textbf{  No. Observations:  } &     1518    \\\\\n",
       "\\textbf{Model:}           &      Logit       & \\textbf{  Df Residuals:      } &     1512    \\\\\n",
       "\\textbf{Method:}          &       MLE        & \\textbf{  Df Model:          } &        5    \\\\\n",
       "\\textbf{Date:}            & Thu, 27 Jun 2024 & \\textbf{  Pseudo R-squ.:     } &  0.08060    \\\\\n",
       "\\textbf{Time:}            &     14:50:52     & \\textbf{  Log-Likelihood:    } &   -454.11   \\\\\n",
       "\\textbf{converged:}       &       True       & \\textbf{  LL-Null:           } &   -493.92   \\\\\n",
       "\\textbf{Covariance Type:} &    nonrobust     & \\textbf{  LLR p-value:       } & 1.009e-15   \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lcccccc}\n",
       "                 & \\textbf{coef} & \\textbf{std err} & \\textbf{z} & \\textbf{P$> |$z$|$} & \\textbf{[0.025} & \\textbf{0.975]}  \\\\\n",
       "\\midrule\n",
       "\\textbf{const}   &       2.4151  &        0.161     &    14.996  &         0.000        &        2.099    &        2.731     \\\\\n",
       "\\textbf{llema}   &      -1.8425  &        0.248     &    -7.431  &         0.000        &       -2.329    &       -1.357     \\\\\n",
       "\\textbf{llama}   &      -0.2697  &        0.277     &    -0.974  &         0.330        &       -0.813    &        0.273     \\\\\n",
       "\\textbf{mammoth} &       0.5149  &        0.349     &     1.476  &         0.140        &       -0.169    &        1.199     \\\\\n",
       "\\textbf{gpt35}   &      -0.2674  &        0.273     &    -0.980  &         0.327        &       -0.802    &        0.268     \\\\\n",
       "\\textbf{gpt4}    &       0.7073  &        0.361     &     1.959  &         0.050        &       -0.000    &        1.415     \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "%\\caption{Logit Regression Results}\n",
       "\\end{center}"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                           Logit Regression Results                           \n",
       "==============================================================================\n",
       "Dep. Variable:               accuracy   No. Observations:                 1518\n",
       "Model:                          Logit   Df Residuals:                     1512\n",
       "Method:                           MLE   Df Model:                            5\n",
       "Date:                Thu, 27 Jun 2024   Pseudo R-squ.:                 0.08060\n",
       "Time:                        14:50:52   Log-Likelihood:                -454.11\n",
       "converged:                       True   LL-Null:                       -493.92\n",
       "Covariance Type:            nonrobust   LLR p-value:                 1.009e-15\n",
       "==============================================================================\n",
       "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "const          2.4151      0.161     14.996      0.000       2.099       2.731\n",
       "llema         -1.8425      0.248     -7.431      0.000      -2.329      -1.357\n",
       "llama         -0.2697      0.277     -0.974      0.330      -0.813       0.273\n",
       "mammoth        0.5149      0.349      1.476      0.140      -0.169       1.199\n",
       "gpt35         -0.2674      0.273     -0.980      0.327      -0.802       0.268\n",
       "gpt4           0.7073      0.361      1.959      0.050      -0.000       1.415\n",
       "==============================================================================\n",
       "\"\"\""
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_df = pd.concat([df, pd.get_dummies(df['model'], dtype = int)], axis=1)\n",
    "reg_df = reg_df[reg_df['solvability']==1]\n",
    "x = reg_df[['llema', 'llama', 'mammoth', 'gpt35', 'gpt4']]\n",
    "x = sm.add_constant(x)\n",
    "y = reg_df['accuracy']\n",
    "log_reg = sm.Logit(y, x).fit() \n",
    "log_reg.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "e5d8c947-fb3f-4c97-bf5e-cf333b1c8449",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.292783\n",
      "         Iterations 7\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>Logit Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>       <td>accuracy</td>     <th>  No. Observations:  </th>  <td>  1518</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                 <td>Logit</td>      <th>  Df Residuals:      </th>  <td>  1506</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>                 <td>MLE</td>       <th>  Df Model:          </th>  <td>    11</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>            <td>Thu, 27 Jun 2024</td> <th>  Pseudo R-squ.:     </th>  <td>0.1002</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                <td>14:50:52</td>     <th>  Log-Likelihood:    </th> <td> -444.44</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>converged:</th>             <td>True</td>       <th>  LL-Null:           </th> <td> -493.92</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>     <td>nonrobust</td>    <th>  LLR p-value:       </th> <td>2.894e-16</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "         <td></td>           <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>          <td>    2.0985</td> <td>    0.263</td> <td>    7.990</td> <td> 0.000</td> <td>    1.584</td> <td>    2.613</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>addition</th>       <td>    0.4167</td> <td>    0.207</td> <td>    2.017</td> <td> 0.044</td> <td>    0.012</td> <td>    0.822</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>subtraction</th>    <td>    0.3226</td> <td>    0.226</td> <td>    1.429</td> <td> 0.153</td> <td>   -0.120</td> <td>    0.765</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>multiplication</th> <td>    0.0086</td> <td>    0.217</td> <td>    0.040</td> <td> 0.968</td> <td>   -0.418</td> <td>    0.435</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>division</th>       <td>   -0.3004</td> <td>    0.245</td> <td>   -1.228</td> <td> 0.219</td> <td>   -0.780</td> <td>    0.179</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>fractions</th>      <td>   -0.0015</td> <td>    0.397</td> <td>   -0.004</td> <td> 0.997</td> <td>   -0.779</td> <td>    0.776</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>decimals</th>       <td>   -0.2565</td> <td>    0.256</td> <td>   -1.001</td> <td> 0.317</td> <td>   -0.758</td> <td>    0.245</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>llema</th>          <td>   -1.6408</td> <td>    0.258</td> <td>   -6.366</td> <td> 0.000</td> <td>   -2.146</td> <td>   -1.136</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>llama</th>          <td>   -0.2501</td> <td>    0.280</td> <td>   -0.892</td> <td> 0.372</td> <td>   -0.800</td> <td>    0.300</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>mammoth</th>        <td>    0.6945</td> <td>    0.356</td> <td>    1.953</td> <td> 0.051</td> <td>   -0.002</td> <td>    1.391</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gpt35</th>          <td>    0.0356</td> <td>    0.284</td> <td>    0.125</td> <td> 0.900</td> <td>   -0.522</td> <td>    0.593</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gpt4</th>           <td>    0.7972</td> <td>    0.373</td> <td>    2.137</td> <td> 0.033</td> <td>    0.066</td> <td>    1.528</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/latex": [
       "\\begin{center}\n",
       "\\begin{tabular}{lclc}\n",
       "\\toprule\n",
       "\\textbf{Dep. Variable:}   &     accuracy     & \\textbf{  No. Observations:  } &     1518    \\\\\n",
       "\\textbf{Model:}           &      Logit       & \\textbf{  Df Residuals:      } &     1506    \\\\\n",
       "\\textbf{Method:}          &       MLE        & \\textbf{  Df Model:          } &       11    \\\\\n",
       "\\textbf{Date:}            & Thu, 27 Jun 2024 & \\textbf{  Pseudo R-squ.:     } &   0.1002    \\\\\n",
       "\\textbf{Time:}            &     14:50:52     & \\textbf{  Log-Likelihood:    } &   -444.44   \\\\\n",
       "\\textbf{converged:}       &       True       & \\textbf{  LL-Null:           } &   -493.92   \\\\\n",
       "\\textbf{Covariance Type:} &    nonrobust     & \\textbf{  LLR p-value:       } & 2.894e-16   \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lcccccc}\n",
       "                        & \\textbf{coef} & \\textbf{std err} & \\textbf{z} & \\textbf{P$> |$z$|$} & \\textbf{[0.025} & \\textbf{0.975]}  \\\\\n",
       "\\midrule\n",
       "\\textbf{const}          &       2.0985  &        0.263     &     7.990  &         0.000        &        1.584    &        2.613     \\\\\n",
       "\\textbf{addition}       &       0.4167  &        0.207     &     2.017  &         0.044        &        0.012    &        0.822     \\\\\n",
       "\\textbf{subtraction}    &       0.3226  &        0.226     &     1.429  &         0.153        &       -0.120    &        0.765     \\\\\n",
       "\\textbf{multiplication} &       0.0086  &        0.217     &     0.040  &         0.968        &       -0.418    &        0.435     \\\\\n",
       "\\textbf{division}       &      -0.3004  &        0.245     &    -1.228  &         0.219        &       -0.780    &        0.179     \\\\\n",
       "\\textbf{fractions}      &      -0.0015  &        0.397     &    -0.004  &         0.997        &       -0.779    &        0.776     \\\\\n",
       "\\textbf{decimals}       &      -0.2565  &        0.256     &    -1.001  &         0.317        &       -0.758    &        0.245     \\\\\n",
       "\\textbf{llema}          &      -1.6408  &        0.258     &    -6.366  &         0.000        &       -2.146    &       -1.136     \\\\\n",
       "\\textbf{llama}          &      -0.2501  &        0.280     &    -0.892  &         0.372        &       -0.800    &        0.300     \\\\\n",
       "\\textbf{mammoth}        &       0.6945  &        0.356     &     1.953  &         0.051        &       -0.002    &        1.391     \\\\\n",
       "\\textbf{gpt35}          &       0.0356  &        0.284     &     0.125  &         0.900        &       -0.522    &        0.593     \\\\\n",
       "\\textbf{gpt4}           &       0.7972  &        0.373     &     2.137  &         0.033        &        0.066    &        1.528     \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "%\\caption{Logit Regression Results}\n",
       "\\end{center}"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                           Logit Regression Results                           \n",
       "==============================================================================\n",
       "Dep. Variable:               accuracy   No. Observations:                 1518\n",
       "Model:                          Logit   Df Residuals:                     1506\n",
       "Method:                           MLE   Df Model:                           11\n",
       "Date:                Thu, 27 Jun 2024   Pseudo R-squ.:                  0.1002\n",
       "Time:                        14:50:52   Log-Likelihood:                -444.44\n",
       "converged:                       True   LL-Null:                       -493.92\n",
       "Covariance Type:            nonrobust   LLR p-value:                 2.894e-16\n",
       "==================================================================================\n",
       "                     coef    std err          z      P>|z|      [0.025      0.975]\n",
       "----------------------------------------------------------------------------------\n",
       "const              2.0985      0.263      7.990      0.000       1.584       2.613\n",
       "addition           0.4167      0.207      2.017      0.044       0.012       0.822\n",
       "subtraction        0.3226      0.226      1.429      0.153      -0.120       0.765\n",
       "multiplication     0.0086      0.217      0.040      0.968      -0.418       0.435\n",
       "division          -0.3004      0.245     -1.228      0.219      -0.780       0.179\n",
       "fractions         -0.0015      0.397     -0.004      0.997      -0.779       0.776\n",
       "decimals          -0.2565      0.256     -1.001      0.317      -0.758       0.245\n",
       "llema             -1.6408      0.258     -6.366      0.000      -2.146      -1.136\n",
       "llama             -0.2501      0.280     -0.892      0.372      -0.800       0.300\n",
       "mammoth            0.6945      0.356      1.953      0.051      -0.002       1.391\n",
       "gpt35              0.0356      0.284      0.125      0.900      -0.522       0.593\n",
       "gpt4               0.7972      0.373      2.137      0.033       0.066       1.528\n",
       "==================================================================================\n",
       "\"\"\""
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = reg_df[['addition', 'subtraction',\n",
    "       'multiplication', 'division', 'fractions', 'decimals', \n",
    "       'llema', 'llama', 'mammoth', 'gpt35', 'gpt4']]\n",
    "y = reg_df['accuracy']\n",
    "x = sm.add_constant(x)\n",
    "log_reg = sm.Logit(y, x).fit() \n",
    "log_reg.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e88714c2-27cf-45f3-8f58-78426f7909e2",
   "metadata": {},
   "source": [
    "## Explore Accuracy by Question Type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "6c46ce60-1563-4988-8f71-3aa185a9d7d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Addition: 0.9603174603174603, Subtraction: 0.9433962264150944, Multiplication: 0.958904109589041, Division: 0.9333333333333333, Decimals: 0.9090909090909091, Fractions: 1.0\n"
     ]
    }
   ],
   "source": [
    "addition_acc = gpt4[gpt4['addition']==1]['accuracy'].mean()\n",
    "subtraction_acc = gpt4[gpt4['subtraction']==1]['accuracy'].mean()\n",
    "multiplication_acc = gpt4[gpt4['multiplication']==1]['accuracy'].mean()\n",
    "division_acc = gpt4[gpt4['division']==1]['accuracy'].mean()\n",
    "decimals_acc = gpt4[gpt4['decimals']==1]['accuracy'].mean()\n",
    "fractions_acc = gpt4[gpt4['fractions']==1]['accuracy'].mean()\n",
    "print(f\"Addition: {addition_acc}, Subtraction: {subtraction_acc}, Multiplication: {multiplication_acc}, Division: {division_acc}, Decimals: {decimals_acc}, Fractions: {fractions_acc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "53a0a0f1-d920-4d2b-9ac7-7051c9973902",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 0.4113766756037213\n",
      "P-value: 0.688697536135214\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/bryan/.pyenv/versions/3.11.5/lib/python3.11/site-packages/scipy/stats/_axis_nan_policy.py:531: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  res = hypotest_fun_out(*samples, **kwds)\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = gpt4[gpt4['fractions']==1]['accuracy'].dropna()\n",
    "low = gpt4[gpt4['decimals']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "abf3565f-2cd3-4a42-8a37-c8f3bee8c3b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Addition: 0.9102564102564102, Subtraction: 0.9523809523809523, Multiplication: 0.865979381443299, Division: 0.875, Decimals: 0.9019607843137255, Fractions: 0.7142857142857143\n"
     ]
    }
   ],
   "source": [
    "addition_acc = gpt35[gpt35['addition']==1]['accuracy'].mean()\n",
    "subtraction_acc = gpt35[gpt35['subtraction']==1]['accuracy'].mean()\n",
    "multiplication_acc = gpt35[gpt35['multiplication']==1]['accuracy'].mean()\n",
    "division_acc = gpt35[gpt35['division']==1]['accuracy'].mean()\n",
    "decimals_acc = gpt35[gpt35['decimals']==1]['accuracy'].mean()\n",
    "fractions_acc = gpt35[gpt35['fractions']==1]['accuracy'].mean()\n",
    "print(f\"Addition: {addition_acc}, Subtraction: {subtraction_acc}, Multiplication: {multiplication_acc}, Division: {division_acc}, Decimals: {decimals_acc}, Fractions: {fractions_acc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "df870cda-8fde-423f-9584-19f9047c84a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 2.3804761428476158\n",
      "P-value: 0.02009873897122017\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = gpt35[gpt35['subtraction']==1]['accuracy'].dropna()\n",
    "low = gpt35[gpt35['fractions']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "3da42528-caf1-4b95-a625-ec81232d7206",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Addition: 0.9612903225806452, Subtraction: 0.974025974025974, Multiplication: 0.9454545454545454, Division: 0.9130434782608695, Decimals: 0.9411764705882353, Fractions: 1.0\n"
     ]
    }
   ],
   "source": [
    "addition_acc = mathwell[mathwell['addition']==1]['accuracy'].mean()\n",
    "subtraction_acc = mathwell[mathwell['subtraction']==1]['accuracy'].mean()\n",
    "multiplication_acc = mathwell[mathwell['multiplication']==1]['accuracy'].mean()\n",
    "division_acc = mathwell[mathwell['division']==1]['accuracy'].mean()\n",
    "decimals_acc = mathwell[mathwell['decimals']==1]['accuracy'].mean()\n",
    "fractions_acc = mathwell[mathwell['fractions']==1]['accuracy'].mean()\n",
    "print(f\"Addition: {addition_acc}, Subtraction: {subtraction_acc}, Multiplication: {multiplication_acc}, Division: {division_acc}, Decimals: {decimals_acc}, Fractions: {fractions_acc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "e16990a6-0249-44aa-814e-7e8c86d617b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 1.5086228956282743\n",
      "P-value: 0.13319844379341608\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = mathwell[mathwell['subtraction']==1]['accuracy'].dropna()\n",
    "low = mathwell[mathwell['division']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "83b393c3-319c-4c6c-b303-9a0fb84f29cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Addition: 0.8925619834710744, Subtraction: 0.9105691056910569, Multiplication: 0.875, Division: 0.8, Decimals: 0.75, Fractions: 0.8235294117647058\n"
     ]
    }
   ],
   "source": [
    "addition_acc = llama[llama['addition']==1]['accuracy'].mean()\n",
    "subtraction_acc = llama[llama['subtraction']==1]['accuracy'].mean()\n",
    "multiplication_acc = llama[llama['multiplication']==1]['accuracy'].mean()\n",
    "division_acc = llama[llama['division']==1]['accuracy'].mean()\n",
    "decimals_acc = llama[llama['decimals']==1]['accuracy'].mean()\n",
    "fractions_acc = llama[llama['fractions']==1]['accuracy'].mean()\n",
    "print(f\"Addition: {addition_acc}, Subtraction: {subtraction_acc}, Multiplication: {multiplication_acc}, Division: {division_acc}, Decimals: {decimals_acc}, Fractions: {fractions_acc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "cea72230-5776-4dbd-9b7d-089d0683c621",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 2.274144870913362\n",
      "P-value: 0.02442499023857366\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = llama[llama['subtraction']==1]['accuracy'].dropna()\n",
    "low = llama[llama['decimals']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "d4db6dd2-e8dd-4afd-9860-0c51163343b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Addition: 0.7619047619047619, Subtraction: 0.7272727272727273, Multiplication: 0.6341463414634146, Division: 0.56, Decimals: 0.631578947368421, Fractions: 0.5\n"
     ]
    }
   ],
   "source": [
    "addition_acc = llema[llema['addition']==1]['accuracy'].mean()\n",
    "subtraction_acc = llema[llema['subtraction']==1]['accuracy'].mean()\n",
    "multiplication_acc = llema[llema['multiplication']==1]['accuracy'].mean()\n",
    "division_acc = llema[llema['division']==1]['accuracy'].mean()\n",
    "decimals_acc = llema[llema['decimals']==1]['accuracy'].mean()\n",
    "fractions_acc = llema[llema['fractions']==1]['accuracy'].mean()\n",
    "print(f\"Addition: {addition_acc}, Subtraction: {subtraction_acc}, Multiplication: {multiplication_acc}, Division: {division_acc}, Decimals: {decimals_acc}, Fractions: {fractions_acc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "4f47b6df-573a-422a-a5b8-a5a8f5de9b24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 1.5166403726005646\n",
      "P-value: 0.13591628815835768\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = llema[llema['addition']==1]['accuracy'].dropna()\n",
    "low = llema[llema['fractions']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "66eec193-8e01-4df2-a7c2-024689b24b35",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Addition: 0.9651162790697675, Subtraction: 0.9634146341463414, Multiplication: 0.968421052631579, Division: 0.9761904761904762, Decimals: 0.9130434782608695, Fractions: 0.875\n"
     ]
    }
   ],
   "source": [
    "addition_acc = mammoth[mammoth['addition']==1]['accuracy'].mean()\n",
    "subtraction_acc = mammoth[mammoth['subtraction']==1]['accuracy'].mean()\n",
    "multiplication_acc = mammoth[mammoth['multiplication']==1]['accuracy'].mean()\n",
    "division_acc = mammoth[mammoth['division']==1]['accuracy'].mean()\n",
    "decimals_acc = mammoth[mammoth['decimals']==1]['accuracy'].mean()\n",
    "fractions_acc = mammoth[mammoth['fractions']==1]['accuracy'].mean()\n",
    "print(f\"Addition: {addition_acc}, Subtraction: {subtraction_acc}, Multiplication: {multiplication_acc}, Division: {division_acc}, Decimals: {decimals_acc}, Fractions: {fractions_acc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "38091b76-190e-43d7-b161-7fa9019f8327",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 1.3357320331221683\n",
      "P-value: 0.1879360655573076\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = mammoth[mammoth['division']==1]['accuracy'].dropna()\n",
    "low = mammoth[mammoth['fractions']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "f58bb02e-39f4-4d17-815f-e0ce664173ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 0.14945453514121473\n",
      "P-value: 0.8813210204814809\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = mammoth[mammoth['addition']==1]['accuracy'].dropna()\n",
    "low = mathwell[mathwell['addition']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "4138d970-67f2-41c8-b76f-b1c0da13d064",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: -0.4557969159501152\n",
      "P-value: 0.6489587131238368\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = mammoth[mammoth['subtraction']==1]['accuracy'].dropna()\n",
    "low = mathwell[mathwell['subtraction']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "0f4c6e91-543c-4fae-bd1f-87a123d9443e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 0.6881860213634111\n",
      "P-value: 0.4924128827161959\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = mammoth[mammoth['multiplication']==1]['accuracy'].dropna()\n",
    "low = mathwell[mathwell['multiplication']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "3577aa7f-fd83-453d-92cf-aa86b6eed2de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 1.154248459318687\n",
      "P-value: 0.25275810709158303\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = mammoth[mammoth['division']==1]['accuracy'].dropna()\n",
    "low = mathwell[mathwell['division']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "e5867960-f016-4c09-9b59-a7eddb176c2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: -1.2421180068162376\n",
      "P-value: 0.23013377921316114\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = mammoth[mammoth['fractions']==1]['accuracy'].dropna()\n",
    "low = mathwell[mathwell['fractions']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "0cbe69c3-adb7-47b9-90c0-8054d1b981ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: -0.3259423987758857\n",
      "P-value: 0.7462561459262849\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = mammoth[mammoth['decimals']==1]['accuracy'].dropna()\n",
    "low = mathwell[mathwell['decimals']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "ba67cceb-bc5a-423b-a146-a7b8e0ab910b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Addition: 0.9047619047619048, Subtraction: 0.8421052631578947, Multiplication: 0.75, Division: 0.7083333333333334, Decimals: 0.8181818181818182, Fractions: 1.0\n"
     ]
    }
   ],
   "source": [
    "addition_acc = llama_gsm[llama_gsm['addition']==1]['accuracy'].mean()\n",
    "subtraction_acc = llama_gsm[llama_gsm['subtraction']==1]['accuracy'].mean()\n",
    "multiplication_acc = llama_gsm[llama_gsm['multiplication']==1]['accuracy'].mean()\n",
    "division_acc = llama_gsm[llama_gsm['division']==1]['accuracy'].mean()\n",
    "decimals_acc = llama_gsm[llama_gsm['decimals']==1]['accuracy'].mean()\n",
    "fractions_acc = llama_gsm[llama_gsm['fractions']==1]['accuracy'].mean()\n",
    "print(f\"Addition: {addition_acc}, Subtraction: {subtraction_acc}, Multiplication: {multiplication_acc}, Division: {division_acc}, Decimals: {decimals_acc}, Fractions: {fractions_acc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "13dccf10-fb38-491c-9280-bbb270b6765a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 1.6569992269143405\n",
      "P-value: 0.1047994333017134\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = llama_gsm[llama_gsm['addition']==1]['accuracy'].dropna()\n",
    "low = llama_gsm[llama_gsm['division']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "1dd8d1e9-da3e-47d5-9615-7517b4ace3a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Addition: 0.948905109489051, Subtraction: 0.9241379310344827, Multiplication: 0.9090909090909091, Division: 0.8260869565217391, Decimals: 0.8260869565217391, Fractions: 1.0\n"
     ]
    }
   ],
   "source": [
    "addition_acc = egsm_only[egsm_only['addition']==1]['accuracy'].mean()\n",
    "subtraction_acc = egsm_only[egsm_only['subtraction']==1]['accuracy'].mean()\n",
    "multiplication_acc = egsm_only[egsm_only['multiplication']==1]['accuracy'].mean()\n",
    "division_acc = egsm_only[egsm_only['division']==1]['accuracy'].mean()\n",
    "decimals_acc = egsm_only[egsm_only['decimals']==1]['accuracy'].mean()\n",
    "fractions_acc = egsm_only[egsm_only['fractions']==1]['accuracy'].mean()\n",
    "print(f\"Addition: {addition_acc}, Subtraction: {subtraction_acc}, Multiplication: {multiplication_acc}, Division: {division_acc}, Decimals: {decimals_acc}, Fractions: {fractions_acc}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "c4a46395-b8a4-4d2c-8f7d-b98204a96d77",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-statistic: 0.8484361389687286\n",
      "P-value: 0.39691985200959146\n"
     ]
    }
   ],
   "source": [
    "# Perform an independent two-sample t-test on highest versus lowest accuracy operation\n",
    "high = egsm_only[egsm_only['addition']==1]['accuracy'].dropna()\n",
    "low = egsm_only[egsm_only['subtraction']==1]['accuracy'].dropna()\n",
    "t_statistic, p_value = stats.ttest_ind(high, low)\n",
    "\n",
    "# Print the results\n",
    "print(\"T-statistic:\", t_statistic)\n",
    "print(\"P-value:\", p_value)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18f2a0d1-334d-413d-bc63-c320605a8969",
   "metadata": {},
   "source": [
    "## Merge DF with Question Data and Send to CSV"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efe2c79a-78b7-4368-8de0-bf3af87bdc8d",
   "metadata": {},
   "source": [
    "questions = pd.read_csv('data/zooniverse_questions.csv')\n",
    "questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "f2b65c60-a596-4730-82b2-e7018d2abfda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "question_number\n",
       "llama_q1053       1\n",
       "gpt35_q2122       1\n",
       "gpt35_q1995       1\n",
       "gpt35_q838        1\n",
       "gpt35_q2205       1\n",
       "                 ..\n",
       "mammoth_q6        1\n",
       "mammoth_q95       1\n",
       "mammoth_q298      1\n",
       "llema_q649        1\n",
       "egsm_only_q127    1\n",
       "Name: count, Length: 1750, dtype: int64"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.to_csv('data/llama_gsm_evaluations.csv')\n",
    "df = df[df['model']!=\"llama_gsm\"]\n",
    "df['question_number'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06978ecf-ef2b-485c-82be-23f47d3b3c2f",
   "metadata": {},
   "source": [
    "merged = pd.merge(df, questions,\n",
    "              on = 'question_number',\n",
    "              how='inner',\n",
    "              validate='one_to_one', \n",
    "              indicator = 'matched')\n",
    "merged['matched'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25f2bf8c-dbea-400d-9bc1-64cf4b4603b4",
   "metadata": {},
   "source": [
    "merged = pd.merge(df, questions,\n",
    "              on = 'question_number',\n",
    "              how='inner')\n",
    "merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "8ac62f04-ae13-4365-958a-4f920bd99c1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "merged['text'] = \"Question: \" + merged['question'] + \"\\nAnswer: \" + merged['answer'].astype('string')\n",
    "merged.to_csv('data/all_models.csv')\n",
    "merged['acc_text'] = \"Question: \" + merged['question'] + \"\\nSolution:\\n\" + merged['solution']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbaef8f4-f7ef-4ff4-8da5-f5d814aa9727",
   "metadata": {},
   "source": [
    "finetune = pd.read_csv('data/finetuning_data.csv')\n",
    "filtered_df = merged[~merged['question'].isin(finetune['question'])]\n",
    "llama = filtered_df[filtered_df['model']=='llama']\n",
    "llema = filtered_df[filtered_df['model']=='llema']\n",
    "mathwell = filtered_df[filtered_df['model']=='mathwell']\n",
    "mammoth = filtered_df[filtered_df['model']=='mammoth']\n",
    "print(f\"Mathwell: {mathwell['good'].mean()}, Llama: {llama['good'].mean()}, Llema: {llema['good'].mean()}, Mammoth: {mammoth['good'].mean()}\")\n",
    "print(f\"Mathwell: {mathwell['solvability'].mean()}, Llama: {llama['solvability'].mean()}, Llema: {llema['solvability'].mean()}, Mammoth: {mammoth['solvability'].mean()}\")\n",
    "print(f\"Mathwell: {mathwell['accuracy'].mean()}, Llama: {llama['accuracy'].mean()}, Llema: {llema['accuracy'].mean()}, Mammoth: {mammoth['accuracy'].mean()}\")\n",
    "print(f\"Mathwell: {mathwell['appropriateness'].mean()}, Llama: {llama['appropriateness'].mean()}, Llema: {llema['appropriateness'].mean()}, Mammoth: {mammoth['appropriateness'].mean()}\")\n",
    "filtered_df.to_csv('data/filtered_models.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "86850a29-fbc2-4df4-b71e-c078e98c9e3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "llama = merged[merged['model']=='llama']\n",
    "llema = merged[merged['model']=='llema']\n",
    "mathwell = merged[merged['model']=='mathwell']\n",
    "mathwell[mathwell['good']==1].to_csv(\"mathwell_good_qs.csv\")\n",
    "mammoth = merged[merged['model']=='mammoth']\n",
    "gpt35 = merged[merged['model']=='gpt35']\n",
    "gpt4 = merged[merged['model']=='gpt4']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "ad41b2f0-e954-43a4-b89c-ec30b5b43e52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>In Minecraft, Steve is building a rectangular ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>In Angel Grove, there were 5 Power Rangers bat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Barbie is preparing goodie bags for her birthd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Mrs. Johnson has 12 adult cats and each adult ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>In Minecraft, Steve needs to build a fence aro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>0</td>\n",
       "      <td>Ash has 50 Poké Balls in his backpack. He catc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>0</td>\n",
       "      <td>Jenny has 12 cats. She decides to buy more cat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>0</td>\n",
       "      <td>Samantha has a dog named Max, who loves to col...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>0</td>\n",
       "      <td>A cheerleading squad is performing a routine t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>0</td>\n",
       "      <td>LeBron James scores an average of 27 points pe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0        1  In Minecraft, Steve is building a rectangular ...\n",
       "1        1  In Angel Grove, there were 5 Power Rangers bat...\n",
       "2        1  Barbie is preparing goodie bags for her birthd...\n",
       "3        1  Mrs. Johnson has 12 adult cats and each adult ...\n",
       "4        1  In Minecraft, Steve needs to build a fence aro...\n",
       "..     ...                                                ...\n",
       "245      0  Ash has 50 Poké Balls in his backpack. He catc...\n",
       "246      0  Jenny has 12 cats. She decides to buy more cat...\n",
       "247      0  Samantha has a dog named Max, who loves to col...\n",
       "248      0  A cheerleading squad is performing a routine t...\n",
       "249      0  LeBron James scores an average of 27 points pe...\n",
       "\n",
       "[250 rows x 2 columns]"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "solvability = gpt4[['solvability', 'question']]\n",
    "solvability = solvability.rename({'solvability': 'label', 'question': 'text'}, axis=1)\n",
    "solvability.to_csv('data/gpt4_solvability.csv')\n",
    "solvability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "e5f6c238-d2b8-4f85-8790-2de7d3b9e602",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Thor, the God of Thunder, has 10 lightning bol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Pikachu has 1000 candy pieces. He wants to evo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>In a lacrosse game, the team scores 3 points f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Thor has 2000 hammers. Each hammer weighs 5 po...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>0</td>\n",
       "      <td>In Minecraft, a player needs 3 blocks of iron ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>0</td>\n",
       "      <td>Barbie has 12 different outfits she can wear. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>0</td>\n",
       "      <td>Brandon ordered 3 pizzas for a party. Each piz...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>0</td>\n",
       "      <td>In Fortnite, a player has 10 shield potions. E...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>0</td>\n",
       "      <td>A dog walker is taking care of 9 dogs for the ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0        1  Thor, the God of Thunder, has 10 lightning bol...\n",
       "1        1  Pikachu has 1000 candy pieces. He wants to evo...\n",
       "2        1  In a lacrosse game, the team scores 3 points f...\n",
       "3        1  Thor has 2000 hammers. Each hammer weighs 5 po...\n",
       "4        1                                                NaN\n",
       "..     ...                                                ...\n",
       "245      0  In Minecraft, a player needs 3 blocks of iron ...\n",
       "246      0  Barbie has 12 different outfits she can wear. ...\n",
       "247      0  Brandon ordered 3 pizzas for a party. Each piz...\n",
       "248      0  In Fortnite, a player has 10 shield potions. E...\n",
       "249      0  A dog walker is taking care of 9 dogs for the ...\n",
       "\n",
       "[250 rows x 2 columns]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "solvability = gpt35[['solvability', 'question']]\n",
    "solvability = solvability.rename({'solvability': 'label', 'question': 'text'}, axis=1)\n",
    "solvability.to_csv('data/gpt35_solvability.csv')\n",
    "solvability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "d93903f3-e4a6-4e61-bcb5-3aff8c301def",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Steph Curry has 3000 points. 1000 of them are ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>A commercial airplane takes 12 hours to fly fr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>Taylor Swift has 100 cats. She has 30 more cat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>Fortnite is a battle royale game. 100 players ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0</td>\n",
       "      <td>Patrick Mahomes threw 26 touchdown passes in t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>989</th>\n",
       "      <td>1</td>\n",
       "      <td>The soccer team scored 3 goals in the first ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>992</th>\n",
       "      <td>1</td>\n",
       "      <td>Serena Williams won 23 Grand Slam titles. She ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>993</th>\n",
       "      <td>1</td>\n",
       "      <td>Batman has 200 batarangs. He throws 20 bataran...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>1</td>\n",
       "      <td>Wonder Woman has 1000 Amazonian cookies. \\nShe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>1</td>\n",
       "      <td>Naruto is 16 years old.  He is 4 years older t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0        1  Steph Curry has 3000 points. 1000 of them are ...\n",
       "2        1  A commercial airplane takes 12 hours to fly fr...\n",
       "6        1  Taylor Swift has 100 cats. She has 30 more cat...\n",
       "16       1  Fortnite is a battle royale game. 100 players ...\n",
       "18       0  Patrick Mahomes threw 26 touchdown passes in t...\n",
       "..     ...                                                ...\n",
       "989      1  The soccer team scored 3 goals in the first ha...\n",
       "992      1  Serena Williams won 23 Grand Slam titles. She ...\n",
       "993      1  Batman has 200 batarangs. He throws 20 bataran...\n",
       "995      1  Wonder Woman has 1000 Amazonian cookies. \\nShe...\n",
       "997      1  Naruto is 16 years old.  He is 4 years older t...\n",
       "\n",
       "[250 rows x 2 columns]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "solvability = llama[['solvability', 'question']]\n",
    "solvability = solvability.rename({'solvability': 'label', 'question': 'text'}, axis=1)\n",
    "solvability.to_csv('data/llama_solvability.csv')\n",
    "solvability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "5aefcc53-f3a3-41c0-920b-8e5876e3d33c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>In the original comic book series, the four Tu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>Spiderman has a web-shooter that shoots spider...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>Naruto has 3000 calories of ramen.  He uses 20...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>Wonder Woman has 5 cats. She also has a dog.  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1</td>\n",
       "      <td>Mentos sell for $3.25 per bag. How much do 4 b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940</th>\n",
       "      <td>1</td>\n",
       "      <td>Player1 builds 400 \\nblocks of diamond blocks....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>942</th>\n",
       "      <td>1</td>\n",
       "      <td>A football team has 50 cheerleaders. 4 cheerle...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>945</th>\n",
       "      <td>1</td>\n",
       "      <td>In the 2013/2014 NBA season, Steph Curry made ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>1</td>\n",
       "      <td>A high school lacrosse team has scored the fol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>978</th>\n",
       "      <td>1</td>\n",
       "      <td>Captain America jumped 4 feet in length and 12...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "3        0  In the original comic book series, the four Tu...\n",
       "5        1  Spiderman has a web-shooter that shoots spider...\n",
       "9        0  Naruto has 3000 calories of ramen.  He uses 20...\n",
       "13       1  Wonder Woman has 5 cats. She also has a dog.  ...\n",
       "21       1  Mentos sell for $3.25 per bag. How much do 4 b...\n",
       "..     ...                                                ...\n",
       "940      1  Player1 builds 400 \\nblocks of diamond blocks....\n",
       "942      1  A football team has 50 cheerleaders. 4 cheerle...\n",
       "945      1  In the 2013/2014 NBA season, Steph Curry made ...\n",
       "946      1  A high school lacrosse team has scored the fol...\n",
       "978      1  Captain America jumped 4 feet in length and 12...\n",
       "\n",
       "[250 rows x 2 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "solvability = llema[['solvability', 'question']]\n",
    "solvability = solvability.rename({'solvability': 'label', 'question': 'text'}, axis=1)\n",
    "solvability.to_csv('data/llema_solvability.csv')\n",
    "solvability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "a0d01486-57be-4d94-bd9b-6eacdee81a55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>The soccer team has 15 players. Each player ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>Iron Man has 100 arc reactors. He deploys 20 a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>Iron Man has 5000 dollars. He spends 1000 doll...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>Naruto can run at a top speed of 1200 km/h. He...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0</td>\n",
       "      <td>How many cobblestones are there now?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>914</th>\n",
       "      <td>1</td>\n",
       "      <td>Thor, the God of Thunder, can throw his hammer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>917</th>\n",
       "      <td>1</td>\n",
       "      <td>If a person eats 2 bowls of ice cream every da...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>920</th>\n",
       "      <td>1</td>\n",
       "      <td>The ice cream shop has 6 different flavors. Ea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>941</th>\n",
       "      <td>0</td>\n",
       "      <td>3 dogs ate 12 bones. How many bones did each d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>955</th>\n",
       "      <td>1</td>\n",
       "      <td>Harry Potter has 3000 vibranium-powered punche...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "1        1  The soccer team has 15 players. Each player ha...\n",
       "7        1  Iron Man has 100 arc reactors. He deploys 20 a...\n",
       "8        0  Iron Man has 5000 dollars. He spends 1000 doll...\n",
       "10       1  Naruto can run at a top speed of 1200 km/h. He...\n",
       "17       0               How many cobblestones are there now?\n",
       "..     ...                                                ...\n",
       "914      1  Thor, the God of Thunder, can throw his hammer...\n",
       "917      1  If a person eats 2 bowls of ice cream every da...\n",
       "920      1  The ice cream shop has 6 different flavors. Ea...\n",
       "941      0  3 dogs ate 12 bones. How many bones did each d...\n",
       "955      1  Harry Potter has 3000 vibranium-powered punche...\n",
       "\n",
       "[250 rows x 2 columns]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "solvability = mammoth[['solvability', 'question']]\n",
    "solvability = solvability.rename({'solvability': 'label', 'question': 'text'}, axis=1)\n",
    "solvability.to_csv('data/mammoth_solvability.csv')\n",
    "solvability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "8fa723a4-ef6f-494d-9a1f-5d9547489cc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Captain America has 1200 ryo. He spends 400 ry...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>A 250-car train is 2.5 miles long. How many 25...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3 scoops of ice cream cost $4. \\nHow much does...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Iron Man has 500 gold coins. He can use 10 gol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>0</td>\n",
       "      <td>The soccer team has 20 players. \\n\\n8 of the p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>0</td>\n",
       "      <td>LeBron James is 6'9\" tall. \\nHe's 36 years old...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>0</td>\n",
       "      <td>LeBron James is 6 feet 9 inches tall.  He wear...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>0</td>\n",
       "      <td>A pizza is cut into 12 slices. 3 of the slices...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>0</td>\n",
       "      <td>A Fortnite player has 5000 V-Bucks. They buy 2...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0        1  LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...\n",
       "1        1  Captain America has 1200 ryo. He spends 400 ry...\n",
       "2        1  A 250-car train is 2.5 miles long. How many 25...\n",
       "3        1  3 scoops of ice cream cost $4. \\nHow much does...\n",
       "4        1  Iron Man has 500 gold coins. He can use 10 gol...\n",
       "..     ...                                                ...\n",
       "245      0  The soccer team has 20 players. \\n\\n8 of the p...\n",
       "246      0  LeBron James is 6'9\" tall. \\nHe's 36 years old...\n",
       "247      0  LeBron James is 6 feet 9 inches tall.  He wear...\n",
       "248      0  A pizza is cut into 12 slices. 3 of the slices...\n",
       "249      0  A Fortnite player has 5000 V-Bucks. They buy 2...\n",
       "\n",
       "[250 rows x 2 columns]"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "solvability = mathwell[['solvability', 'question']]\n",
    "solvability = solvability.rename({'solvability': 'label', 'question': 'text'}, axis=1)\n",
    "solvability.to_csv('data/mathwell_solvability.csv')\n",
    "solvability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "891acf26-a041-4449-b6dc-cdabb066ca52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Steph Curry has 3000 points. 1000 of them are ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>The soccer team has 15 players. Each player ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>A commercial airplane takes 12 hours to fly fr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>In the original comic book series, the four Tu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>Spiderman has a web-shooter that shoots spider...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>0</td>\n",
       "      <td>A volleyball team has 1200 points. 1000 of the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>0</td>\n",
       "      <td>A dog has 12 bones. 4 of the bones are in the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>0</td>\n",
       "      <td>The Cedar Rapids Volleyball team has 12 player...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>0</td>\n",
       "      <td>The T-Rex is the largest dinosaur.  It is 1000...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>0</td>\n",
       "      <td>A volleyball team has 12 players. 3 of the pla...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1850 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    label                                               text\n",
       "0       1  Steph Curry has 3000 points. 1000 of them are ...\n",
       "1       1  The soccer team has 15 players. Each player ha...\n",
       "2       1  A commercial airplane takes 12 hours to fly fr...\n",
       "3       0  In the original comic book series, the four Tu...\n",
       "5       1  Spiderman has a web-shooter that shoots spider...\n",
       "..    ...                                                ...\n",
       "95      0  A volleyball team has 1200 points. 1000 of the...\n",
       "96      0  A dog has 12 bones. 4 of the bones are in the ...\n",
       "97      0  The Cedar Rapids Volleyball team has 12 player...\n",
       "98      0  The T-Rex is the largest dinosaur.  It is 1000...\n",
       "99      0  A volleyball team has 12 players. 3 of the pla...\n",
       "\n",
       "[1850 rows x 2 columns]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "solvability = merged[['solvability', 'question']]\n",
    "solvability = solvability.rename({'solvability': 'label', 'question': 'text'}, axis=1)\n",
    "solvability.to_csv('data/solvability_all_models.csv')\n",
    "solvability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "6374d899-7366-4176-920c-790e760570d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Steph Curry has 3000 points. 1000 of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The soccer team has 15 players. Each...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A commercial airplane takes 12 hours...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Spiderman has a web-shooter that sho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Taylor Swift has 100 cats. She has 3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A swimmer swims 1000 meters per hour...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The Ninja Turtles are in a pizza eat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Barbie has 1000 pink outfits. 200 of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: 1200 people were at a soccer game. 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The field hockey team has 15 players...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1517 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    label                                               text\n",
       "0     1.0  Question: Steph Curry has 3000 points. 1000 of...\n",
       "1     1.0  Question: The soccer team has 15 players. Each...\n",
       "2     1.0  Question: A commercial airplane takes 12 hours...\n",
       "5     1.0  Question: Spiderman has a web-shooter that sho...\n",
       "6     0.0  Question: Taylor Swift has 100 cats. She has 3...\n",
       "..    ...                                                ...\n",
       "81    1.0  Question: A swimmer swims 1000 meters per hour...\n",
       "82    1.0  Question: The Ninja Turtles are in a pizza eat...\n",
       "83    1.0  Question: Barbie has 1000 pink outfits. 200 of...\n",
       "84    1.0  Question: 1200 people were at a soccer game. 1...\n",
       "85    1.0  Question: The field hockey team has 15 players...\n",
       "\n",
       "[1517 rows x 2 columns]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "appropriateness = merged[['appropriateness', 'text']].dropna()\n",
    "appropriateness = appropriateness.rename({'appropriateness': 'label'}, axis = 1)\n",
    "appropriateness = appropriateness[['label', 'text']]\n",
    "appropriateness.to_csv('data/appropriateness_all_models.csv')\n",
    "appropriateness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "6c92e5e3-5002-4daa-a5b5-7c37c82604a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: In Minecraft, Steve is building a re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: In Angel Grove, there were 5 Power R...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Barbie is preparing goodie bags for ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Mrs. Johnson has 12 adult cats and e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: In Minecraft, Steve needs to build a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Jenny baked 24 chocolate chip cookie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: During a summer volleyball tournamen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The Teenage Mutant Ninja Turtles are...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Barbie is planning a tea party and s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A car rental company has a fleet of ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>237 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0      0.0  Question: In Minecraft, Steve is building a re...\n",
       "1      0.0  Question: In Angel Grove, there were 5 Power R...\n",
       "2      0.0  Question: Barbie is preparing goodie bags for ...\n",
       "3      0.0  Question: Mrs. Johnson has 12 adult cats and e...\n",
       "4      0.0  Question: In Minecraft, Steve needs to build a...\n",
       "..     ...                                                ...\n",
       "232    1.0  Question: Jenny baked 24 chocolate chip cookie...\n",
       "233    1.0  Question: During a summer volleyball tournamen...\n",
       "234    1.0  Question: The Teenage Mutant Ninja Turtles are...\n",
       "235    1.0  Question: Barbie is planning a tea party and s...\n",
       "236    1.0  Question: A car rental company has a fleet of ...\n",
       "\n",
       "[237 rows x 2 columns]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "appropriateness = gpt4[['appropriateness', 'text']].dropna()\n",
    "appropriateness = appropriateness.rename({'appropriateness': 'label'}, axis = 1)\n",
    "appropriateness.to_csv('data/gpt4_appropriateness.csv')\n",
    "appropriateness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "db8c03ac-f1eb-42e4-89a9-eecbf67599ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Thor, the God of Thunder, has 10 lig...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Pikachu has 1000 candy pieces. He wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: In a lacrosse game, the team scores ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Thor has 2000 hammers. Each hammer w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Iron Man has 6 suits in his closet. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Pikachu has 85 HP. Charmander has 60...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A train travels at a speed of 50 mil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: There are 50 cats in a pet store. 20...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The cheerleading squad has 20 member...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The basketball team practiced shooti...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>219 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0      0.0  Question: Thor, the God of Thunder, has 10 lig...\n",
       "1      0.0  Question: Pikachu has 1000 candy pieces. He wa...\n",
       "2      0.0  Question: In a lacrosse game, the team scores ...\n",
       "3      0.0  Question: Thor has 2000 hammers. Each hammer w...\n",
       "5      0.0  Question: Iron Man has 6 suits in his closet. ...\n",
       "..     ...                                                ...\n",
       "216    1.0  Question: Pikachu has 85 HP. Charmander has 60...\n",
       "217    1.0  Question: A train travels at a speed of 50 mil...\n",
       "218    1.0  Question: There are 50 cats in a pet store. 20...\n",
       "219    1.0  Question: The cheerleading squad has 20 member...\n",
       "220    1.0  Question: The basketball team practiced shooti...\n",
       "\n",
       "[219 rows x 2 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "appropriateness = gpt35[['appropriateness', 'text']].dropna()\n",
    "appropriateness = appropriateness.rename({'appropriateness': 'label'}, axis = 1)\n",
    "appropriateness.to_csv('data/gpt35_appropriateness.csv')\n",
    "appropriateness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "ba4f7398-ed08-4fe7-8f37-125ecc160347",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Steph Curry has 3000 points. 1000 of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A commercial airplane takes 12 hours...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Taylor Swift has 100 cats. She has 3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Fortnite is a battle royale game. 10...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: 30 people are going to a pizza party...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>989</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The soccer team scored 3 goals in th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>992</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Serena Williams won 23 Grand Slam ti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>993</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Batman has 200 batarangs. He throws ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Wonder Woman has 1000 Amazonian cook...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Naruto is 16 years old.  He is 4 yea...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>210 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0      1.0  Question: Steph Curry has 3000 points. 1000 of...\n",
       "2      1.0  Question: A commercial airplane takes 12 hours...\n",
       "6      0.0  Question: Taylor Swift has 100 cats. She has 3...\n",
       "16     1.0  Question: Fortnite is a battle royale game. 10...\n",
       "30     1.0  Question: 30 people are going to a pizza party...\n",
       "..     ...                                                ...\n",
       "989    1.0  Question: The soccer team scored 3 goals in th...\n",
       "992    0.0  Question: Serena Williams won 23 Grand Slam ti...\n",
       "993    1.0  Question: Batman has 200 batarangs. He throws ...\n",
       "995    1.0  Question: Wonder Woman has 1000 Amazonian cook...\n",
       "997    1.0  Question: Naruto is 16 years old.  He is 4 yea...\n",
       "\n",
       "[210 rows x 2 columns]"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "appropriateness = llama[['appropriateness', 'text']].dropna()\n",
    "appropriateness = appropriateness.rename({'appropriateness': 'label'}, axis = 1)\n",
    "appropriateness.to_csv('data/llama_appropriateness.csv')\n",
    "appropriateness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "173b32c9-eb3a-4818-8dd4-a068874cc176",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Spiderman has a web-shooter that sho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Wonder Woman has 5 cats. She also ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Mentos sell for $3.25 per bag. How m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Black Panther has 50000 vibranium co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Mario has 22 coins. He finds 30 more...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Player1 builds 400 \n",
       "blocks of diamon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>942</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A football team has 50 cheerleaders....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>945</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: In the 2013/2014 NBA season, Steph C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: A high school lacrosse team has scor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>978</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Captain America jumped 4 feet in len...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>122 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "5      1.0  Question: Spiderman has a web-shooter that sho...\n",
       "13     1.0  Question: Wonder Woman has 5 cats. She also ha...\n",
       "21     1.0  Question: Mentos sell for $3.25 per bag. How m...\n",
       "41     0.0  Question: Black Panther has 50000 vibranium co...\n",
       "43     0.0  Question: Mario has 22 coins. He finds 30 more...\n",
       "..     ...                                                ...\n",
       "940    0.0  Question: Player1 builds 400 \n",
       "blocks of diamon...\n",
       "942    1.0  Question: A football team has 50 cheerleaders....\n",
       "945    1.0  Question: In the 2013/2014 NBA season, Steph C...\n",
       "946    0.0  Question: A high school lacrosse team has scor...\n",
       "978    0.0  Question: Captain America jumped 4 feet in len...\n",
       "\n",
       "[122 rows x 2 columns]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "appropriateness = llema[['appropriateness', 'text']].dropna()\n",
    "appropriateness = appropriateness.rename({'appropriateness': 'label'}, axis = 1)\n",
    "appropriateness.to_csv('data/llema_appropriateness.csv')\n",
    "appropriateness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "20adb4d7-e68a-4246-bbb8-360fa6e9bc28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The soccer team has 15 players. Each...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Iron Man has 100 arc reactors. He de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Naruto can run at a top speed of 120...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Wonder Woman has 100 arrows and 1000...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A trainer has 600 Pokémon. He releas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>912</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Harry Potter has 200 galleons. He sp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>914</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Thor, the God of Thunder, can throw ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>917</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: If a person eats 2 bowls of ice crea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>920</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: The ice cream shop has 6 different f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>955</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Harry Potter has 3000 vibranium-powe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>217 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "1      1.0  Question: The soccer team has 15 players. Each...\n",
       "7      1.0  Question: Iron Man has 100 arc reactors. He de...\n",
       "10     0.0  Question: Naruto can run at a top speed of 120...\n",
       "22     0.0  Question: Wonder Woman has 100 arrows and 1000...\n",
       "23     1.0  Question: A trainer has 600 Pokémon. He releas...\n",
       "..     ...                                                ...\n",
       "912    0.0  Question: Harry Potter has 200 galleons. He sp...\n",
       "914    1.0  Question: Thor, the God of Thunder, can throw ...\n",
       "917    1.0  Question: If a person eats 2 bowls of ice crea...\n",
       "920    0.0  Question: The ice cream shop has 6 different f...\n",
       "955    1.0  Question: Harry Potter has 3000 vibranium-powe...\n",
       "\n",
       "[217 rows x 2 columns]"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "appropriateness = mammoth[['appropriateness', 'text']].dropna()\n",
    "appropriateness = appropriateness.rename({'appropriateness': 'label'}, axis = 1)\n",
    "appropriateness.to_csv('data/mammoth_appropriateness.csv')\n",
    "appropriateness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "72af378c-26b2-4aa5-aae0-777845b62520",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: LeBron James is 6'9\" tall. 1 inch is...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Captain America has 1200 ryo. He spe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A 250-car train is 2.5 miles long. H...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: 3 scoops of ice cream cost $4. \n",
       "How ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Iron Man has 500 gold coins. He can ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A candy store has 120 lollipops. 30 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Wonder Woman has 100 lassos. She las...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Superman can punch 2000 times per mi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A 25-meter pool is 2.5 times as long...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Serena Williams has 24 Grand Slam ti...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>223 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0      1.0  Question: LeBron James is 6'9\" tall. 1 inch is...\n",
       "1      1.0  Question: Captain America has 1200 ryo. He spe...\n",
       "2      1.0  Question: A 250-car train is 2.5 miles long. H...\n",
       "3      1.0  Question: 3 scoops of ice cream cost $4. \n",
       "How ...\n",
       "4      0.0  Question: Iron Man has 500 gold coins. He can ...\n",
       "..     ...                                                ...\n",
       "218    1.0  Question: A candy store has 120 lollipops. 30 ...\n",
       "219    1.0  Question: Wonder Woman has 100 lassos. She las...\n",
       "220    1.0  Question: Superman can punch 2000 times per mi...\n",
       "221    1.0  Question: A 25-meter pool is 2.5 times as long...\n",
       "231    1.0  Question: Serena Williams has 24 Grand Slam ti...\n",
       "\n",
       "[223 rows x 2 columns]"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "appropriateness = mathwell[['appropriateness', 'text']].dropna()\n",
    "appropriateness = appropriateness.rename({'appropriateness': 'label'}, axis = 1)\n",
    "appropriateness.to_csv('data/mathwell_appropriateness.csv')\n",
    "appropriateness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "9bfcaf8c-d75a-4c66-990d-a6990e8541ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: In Minecraft, Steve is building a re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: In Angel Grove, there were 5 Power R...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Barbie is preparing goodie bags for ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Mrs. Johnson has 12 adult cats and e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: In Minecraft, Steve needs to build a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Jenny baked 24 chocolate chip cookie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: During a summer volleyball tournamen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The Teenage Mutant Ninja Turtles are...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Barbie is planning a tea party and s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A car rental company has a fleet of ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>237 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0      0.0  Question: In Minecraft, Steve is building a re...\n",
       "1      0.0  Question: In Angel Grove, there were 5 Power R...\n",
       "2      0.0  Question: Barbie is preparing goodie bags for ...\n",
       "3      0.0  Question: Mrs. Johnson has 12 adult cats and e...\n",
       "4      0.0  Question: In Minecraft, Steve needs to build a...\n",
       "..     ...                                                ...\n",
       "232    1.0  Question: Jenny baked 24 chocolate chip cookie...\n",
       "233    1.0  Question: During a summer volleyball tournamen...\n",
       "234    1.0  Question: The Teenage Mutant Ninja Turtles are...\n",
       "235    1.0  Question: Barbie is planning a tea party and s...\n",
       "236    1.0  Question: A car rental company has a fleet of ...\n",
       "\n",
       "[237 rows x 2 columns]"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy = gpt4[['accuracy', 'acc_text']]\n",
    "accuracy = accuracy[accuracy['accuracy'].notna()]\n",
    "accuracy = accuracy.rename({'accuracy': 'label', 'acc_text': 'text'}, axis=1)\n",
    "accuracy.to_csv('data/gpt4_accuracy.csv')\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "04416bc5-9f84-4694-a0dd-bc0e8dac53c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Thor, the God of Thunder, has 10 lig...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Pikachu has 1000 candy pieces. He wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: In a lacrosse game, the team scores ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Thor has 2000 hammers. Each hammer w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Pikachu has 85 HP. Charmander has 60...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A train travels at a speed of 50 mil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: There are 50 cats in a pet store. 20...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The cheerleading squad has 20 member...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The basketball team practiced shooti...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>220 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0      1.0  Question: Thor, the God of Thunder, has 10 lig...\n",
       "1      1.0  Question: Pikachu has 1000 candy pieces. He wa...\n",
       "2      1.0  Question: In a lacrosse game, the team scores ...\n",
       "3      0.0  Question: Thor has 2000 hammers. Each hammer w...\n",
       "4      1.0                                                NaN\n",
       "..     ...                                                ...\n",
       "216    1.0  Question: Pikachu has 85 HP. Charmander has 60...\n",
       "217    1.0  Question: A train travels at a speed of 50 mil...\n",
       "218    1.0  Question: There are 50 cats in a pet store. 20...\n",
       "219    1.0  Question: The cheerleading squad has 20 member...\n",
       "220    1.0  Question: The basketball team practiced shooti...\n",
       "\n",
       "[220 rows x 2 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy = gpt35[['accuracy', 'acc_text']]\n",
    "accuracy = accuracy[accuracy['accuracy'].notna()]\n",
    "accuracy = accuracy.rename({'accuracy': 'label', 'acc_text': 'text'}, axis=1)\n",
    "accuracy.to_csv('data/gpt35_accuracy.csv')\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "3faa8334-b312-439e-86a3-072b7b3a2817",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Steph Curry has 3000 points. 1000 of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: A commercial airplane takes 12 hours...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Taylor Swift has 100 cats. She has 3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Fortnite is a battle royale game. 10...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: 30 people are going to a pizza party...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>989</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The soccer team scored 3 goals in th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>992</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Serena Williams won 23 Grand Slam ti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>993</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Batman has 200 batarangs. He throws ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Wonder Woman has 1000 Amazonian cook...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Naruto is 16 years old.  He is 4 yea...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>210 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0      1.0  Question: Steph Curry has 3000 points. 1000 of...\n",
       "2      0.0  Question: A commercial airplane takes 12 hours...\n",
       "6      1.0  Question: Taylor Swift has 100 cats. She has 3...\n",
       "16     1.0  Question: Fortnite is a battle royale game. 10...\n",
       "30     1.0  Question: 30 people are going to a pizza party...\n",
       "..     ...                                                ...\n",
       "989    1.0  Question: The soccer team scored 3 goals in th...\n",
       "992    1.0  Question: Serena Williams won 23 Grand Slam ti...\n",
       "993    1.0  Question: Batman has 200 batarangs. He throws ...\n",
       "995    1.0  Question: Wonder Woman has 1000 Amazonian cook...\n",
       "997    1.0  Question: Naruto is 16 years old.  He is 4 yea...\n",
       "\n",
       "[210 rows x 2 columns]"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy = llama[['accuracy', 'acc_text']]\n",
    "accuracy = accuracy[accuracy['accuracy'].notna()]\n",
    "accuracy = accuracy.rename({'accuracy': 'label', 'acc_text': 'text'}, axis=1)\n",
    "accuracy.to_csv('data/llama_accuracy.csv')\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "c9a6aded-6e36-406a-97ef-97d0f487be58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Spiderman has a web-shooter that sho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Wonder Woman has 5 cats. She also ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Mentos sell for $3.25 per bag. How m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Black Panther has 50000 vibranium co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Mario has 22 coins. He finds 30 more...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: Player1 builds 400 \\nblocks of diamo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>942</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: A football team has 50 cheerleaders....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>945</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: In the 2013/2014 NBA season, Steph C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A high school lacrosse team has scor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>978</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Captain America jumped 4 feet in len...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>122 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "5      1.0  Question: Spiderman has a web-shooter that sho...\n",
       "13     1.0  Question: Wonder Woman has 5 cats. She also ha...\n",
       "21     0.0  Question: Mentos sell for $3.25 per bag. How m...\n",
       "41     1.0  Question: Black Panther has 50000 vibranium co...\n",
       "43     1.0  Question: Mario has 22 coins. He finds 30 more...\n",
       "..     ...                                                ...\n",
       "940    0.0  Question: Player1 builds 400 \\nblocks of diamo...\n",
       "942    0.0  Question: A football team has 50 cheerleaders....\n",
       "945    0.0  Question: In the 2013/2014 NBA season, Steph C...\n",
       "946    1.0  Question: A high school lacrosse team has scor...\n",
       "978    1.0  Question: Captain America jumped 4 feet in len...\n",
       "\n",
       "[122 rows x 2 columns]"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy = llema[['accuracy', 'acc_text']]\n",
    "accuracy = accuracy[accuracy['accuracy'].notna()]\n",
    "accuracy = accuracy.rename({'accuracy': 'label', 'acc_text': 'text'}, axis=1)\n",
    "accuracy.to_csv('data/llema_accuracy.csv')\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "75e4bc0f-6974-4ccf-b838-9c27af152bba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The soccer team has 15 players. Each...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Iron Man has 100 arc reactors. He de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Naruto can run at a top speed of 120...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Wonder Woman has 100 arrows and 1000...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A trainer has 600 Pokémon. He releas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>912</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Harry Potter has 200 galleons. He sp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>914</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Thor, the God of Thunder, can throw ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>917</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: If a person eats 2 bowls of ice crea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>920</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: The ice cream shop has 6 different f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>955</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Harry Potter has 3000 vibranium-powe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>217 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "1      1.0  Question: The soccer team has 15 players. Each...\n",
       "7      1.0  Question: Iron Man has 100 arc reactors. He de...\n",
       "10     1.0  Question: Naruto can run at a top speed of 120...\n",
       "22     1.0  Question: Wonder Woman has 100 arrows and 1000...\n",
       "23     1.0  Question: A trainer has 600 Pokémon. He releas...\n",
       "..     ...                                                ...\n",
       "912    1.0  Question: Harry Potter has 200 galleons. He sp...\n",
       "914    1.0  Question: Thor, the God of Thunder, can throw ...\n",
       "917    1.0  Question: If a person eats 2 bowls of ice crea...\n",
       "920    1.0  Question: The ice cream shop has 6 different f...\n",
       "955    1.0  Question: Harry Potter has 3000 vibranium-powe...\n",
       "\n",
       "[217 rows x 2 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy = mammoth[['accuracy', 'acc_text']]\n",
    "accuracy = accuracy[accuracy['accuracy'].notna()]\n",
    "accuracy = accuracy.rename({'accuracy': 'label', 'acc_text': 'text'}, axis=1)\n",
    "accuracy.to_csv('data/mammoth_accuracy.csv')\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "536efe6e-5c80-4b54-8d99-4b9bab4d488b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: LeBron James is 6'9\" tall. 1 inch is...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Captain America has 1200 ryo. He spe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: A 250-car train is 2.5 miles long. H...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: 3 scoops of ice cream cost $4. \\nHow...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Iron Man has 500 gold coins. He can ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A candy store has 120 lollipops. 30 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Wonder Woman has 100 lassos. She las...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Superman can punch 2000 times per mi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A 25-meter pool is 2.5 times as long...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: Serena Williams has 24 Grand Slam ti...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>223 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                               text\n",
       "0      1.0  Question: LeBron James is 6'9\" tall. 1 inch is...\n",
       "1      1.0  Question: Captain America has 1200 ryo. He spe...\n",
       "2      0.0  Question: A 250-car train is 2.5 miles long. H...\n",
       "3      1.0  Question: 3 scoops of ice cream cost $4. \\nHow...\n",
       "4      1.0  Question: Iron Man has 500 gold coins. He can ...\n",
       "..     ...                                                ...\n",
       "218    1.0  Question: A candy store has 120 lollipops. 30 ...\n",
       "219    1.0  Question: Wonder Woman has 100 lassos. She las...\n",
       "220    1.0  Question: Superman can punch 2000 times per mi...\n",
       "221    1.0  Question: A 25-meter pool is 2.5 times as long...\n",
       "231    1.0  Question: Serena Williams has 24 Grand Slam ti...\n",
       "\n",
       "[223 rows x 2 columns]"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy = mathwell[['accuracy', 'acc_text']]\n",
    "accuracy = accuracy[accuracy['accuracy'].notna()]\n",
    "accuracy = accuracy.rename({'accuracy': 'label', 'acc_text': 'text'}, axis=1)\n",
    "accuracy.to_csv('data/mathwell_accuracy.csv')\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "51e2b941-73aa-497b-bd67-8e46e5a6bcb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Question: Steph Curry has 3000 points. 1000 of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Question: The soccer team has 15 players. Each...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>Question: A commercial airplane takes 12 hours...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>Question: In the original comic book series, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>Question: Spiderman has a web-shooter that sho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>0</td>\n",
       "      <td>Question: A volleyball team has 1200 points. 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>0</td>\n",
       "      <td>Question: A dog has 12 bones. 4 of the bones a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>0</td>\n",
       "      <td>Question: The Cedar Rapids Volleyball team has...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>0</td>\n",
       "      <td>Question: The T-Rex is the largest dinosaur.  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>0</td>\n",
       "      <td>Question: A volleyball team has 12 players. 3 ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1850 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    label                                               text\n",
       "0       1  Question: Steph Curry has 3000 points. 1000 of...\n",
       "1       1  Question: The soccer team has 15 players. Each...\n",
       "2       0  Question: A commercial airplane takes 12 hours...\n",
       "3       0  Question: In the original comic book series, t...\n",
       "5       1  Question: Spiderman has a web-shooter that sho...\n",
       "..    ...                                                ...\n",
       "95      0  Question: A volleyball team has 1200 points. 1...\n",
       "96      0  Question: A dog has 12 bones. 4 of the bones a...\n",
       "97      0  Question: The Cedar Rapids Volleyball team has...\n",
       "98      0  Question: The T-Rex is the largest dinosaur.  ...\n",
       "99      0  Question: A volleyball team has 12 players. 3 ...\n",
       "\n",
       "[1850 rows x 2 columns]"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "good = merged[['good', 'text']]\n",
    "good = good.rename({'good': 'label'}, axis = 1)\n",
    "good.to_csv('data/good_all_models.csv')\n",
    "good"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "c089607a-231d-42a8-8741-fe3bede9577b",
   "metadata": {},
   "outputs": [],
   "source": [
    "good = llama.query('good==1')\n",
    "good = good['question']\n",
    "good.to_csv('data/llama_good.csv')\n",
    "good = llema.query('good==1')\n",
    "good = good['question']\n",
    "good.to_csv('data/llema_good.csv')\n",
    "good = mammoth.query('good==1')\n",
    "good = good['question']\n",
    "good.to_csv('data/mammoth_good.csv')\n",
    "good = mathwell.query('good==1')\n",
    "good = good['question']\n",
    "good.to_csv('data/mathwell_good.csv')\n",
    "good = gpt35.query('good==1')\n",
    "good = good['question']\n",
    "good.to_csv('data/gpt35_good.csv')\n",
    "good = gpt4.query('good==1')\n",
    "good = good['question']\n",
    "good.to_csv('data/gpt4_good.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "bd00eba7-719c-4173-88fe-835109f755a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A cat has 9 lives. 3 of them are for eating, 3 of them are for sleeping, and 3 of them are for playing. How many lives does the cat have left for other activities?\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "def sample_question(model, criteria, value):\n",
    "    print(model[model[criteria]==value].iloc[random.randint(0, len(model[model[criteria]==value])-1)]['question'])\n",
    "    \n",
    "def sample_question_solution(model, criteria, value):\n",
    "    index = random.randint(0, len(model[model[criteria]==value])-1)\n",
    "    print(\"Question: \" + model[model[criteria]==value].iloc[index]['question'] + \"\\n\\nSolution:\\n\" + model[model[criteria]==value].iloc[index]['solution'])\n",
    "sample_question(mathwell, criteria = 'appropriateness', value =0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "9d37adf9-fa3e-4499-a2fa-bc58444884b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A dog rescue shelter has 35 dogs, with 15 of them being puppies and the rest being adult dogs. Each adult dog eats 2 cups of dog food per day and each puppy eats 1 cup of dog food per day. If the shelter has a 50-pound bag of dog food, how many days will it last for all the dogs?\n"
     ]
    }
   ],
   "source": [
    "sample_question(gpt35, 'solvability',0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "8e801939-8498-43cd-ace2-68784370a29b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: Iron Man has defeated 8 villains in his first month of crime-fighting. The next month, he defeats 12 villains. If Iron Man defeats 15% more villains each month, how many villains will Iron Man defeat in his 6th month of crime-fighting?\n",
      "\n",
      "Solution:\n",
      "def solution():\n",
      "    #Iron Man defeated 8 villains in his first month\n",
      "    month_1 = 8\n",
      "    #In his second month, he defeated 12 villains\n",
      "    month_2 = 12\n",
      "    #Each month, Iron Man defeats 15% more villains\n",
      "    monthly_increase = 1.15\n",
      "    #Iron Man is in his sixth month\n",
      "    current_month = 6\n",
      "    #Calculate the number of villains in the sixth month\n",
      "    villain_count = month_1 + month_2\n",
      "    for i in range(2, current_month):\n",
      "        villain_count *= monthly_increase\n",
      "    return villain_count\n"
     ]
    }
   ],
   "source": [
    "sample_question_solution(gpt35, 'accuracy', 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7eb1738-f876-429f-8ed5-49a042afdf0b",
   "metadata": {},
   "source": [
    "## Create Annotated Dataset for All Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "b05fa817-7196-4808-b6b0-7b85606bd604",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>solvability</th>\n",
       "      <th>appropriateness</th>\n",
       "      <th>good</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>llama</td>\n",
       "      <td>Steph Curry has 3000 points. 1000 of them are ...</td>\n",
       "      <td>def solution():\\n    #Steph Curry has 3000 poi...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mammoth</td>\n",
       "      <td>The soccer team has 15 players. Each player ha...</td>\n",
       "      <td>def solution():\\n    #Number of players\\n    p...</td>\n",
       "      <td>30</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>llama</td>\n",
       "      <td>A commercial airplane takes 12 hours to fly fr...</td>\n",
       "      <td>def solution():\\n    #A commercial airplane ta...</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>llema</td>\n",
       "      <td>In the original comic book series, the four Tu...</td>\n",
       "      <td>def solution():\\n    #Raphael is 6 inches tall...</td>\n",
       "      <td>30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>llema</td>\n",
       "      <td>Spiderman has a web-shooter that shoots spider...</td>\n",
       "      <td>def solution():\\n    #Spiderman needs to load ...</td>\n",
       "      <td>180</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3229</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>A field hockey game has 11 players per team. T...</td>\n",
       "      <td>def solution():\\n    #A field hockey game has ...</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>100 turtles are at the pond. 10 turtles are ea...</td>\n",
       "      <td>def solution():\\n    #Total turtles\\n    total...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3231</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>Barbie has 1200 Instagram followers. 200 more ...</td>\n",
       "      <td>def solution():\\n    #Barbie started with 1200...</td>\n",
       "      <td>1150</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3232</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>1234567890 is a 10-digit number. \\nHow many 3-...</td>\n",
       "      <td>def solution():\\n    #10-digit number has 10 d...</td>\n",
       "      <td>3.3333333333333335</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3233</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>7 cats and 23 dogs are in a cage. 14 of the an...</td>\n",
       "      <td>def solution():\\n    #Total cats and dogs\\n   ...</td>\n",
       "      <td>13</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5084 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                model                                           question  \\\n",
       "0               llama  Steph Curry has 3000 points. 1000 of them are ...   \n",
       "1             mammoth  The soccer team has 15 players. Each player ha...   \n",
       "2               llama  A commercial airplane takes 12 hours to fly fr...   \n",
       "3               llema  In the original comic book series, the four Tu...   \n",
       "5               llema  Spiderman has a web-shooter that shoots spider...   \n",
       "...               ...                                                ...   \n",
       "3229  finetuned_llama  A field hockey game has 11 players per team. T...   \n",
       "3230  finetuned_llama  100 turtles are at the pond. 10 turtles are ea...   \n",
       "3231  finetuned_llama  Barbie has 1200 Instagram followers. 200 more ...   \n",
       "3232  finetuned_llama  1234567890 is a 10-digit number. \\nHow many 3-...   \n",
       "3233  finetuned_llama  7 cats and 23 dogs are in a cage. 14 of the an...   \n",
       "\n",
       "                                               solution              answer  \\\n",
       "0     def solution():\\n    #Steph Curry has 3000 poi...                   0   \n",
       "1     def solution():\\n    #Number of players\\n    p...                  30   \n",
       "2     def solution():\\n    #A commercial airplane ta...                 0.6   \n",
       "3     def solution():\\n    #Raphael is 6 inches tall...                  30   \n",
       "5     def solution():\\n    #Spiderman needs to load ...                 180   \n",
       "...                                                 ...                 ...   \n",
       "3229  def solution():\\n    #A field hockey game has ...                  33   \n",
       "3230  def solution():\\n    #Total turtles\\n    total...                   0   \n",
       "3231  def solution():\\n    #Barbie started with 1200...                1150   \n",
       "3232  def solution():\\n    #10-digit number has 10 d...  3.3333333333333335   \n",
       "3233  def solution():\\n    #Total cats and dogs\\n   ...                  13   \n",
       "\n",
       "      accuracy  solvability  appropriateness  good  \n",
       "0          1.0            1              1.0     1  \n",
       "1          1.0            1              1.0     1  \n",
       "2          0.0            1              1.0     0  \n",
       "3          NaN            0              NaN     0  \n",
       "5          1.0            1              1.0     1  \n",
       "...        ...          ...              ...   ...  \n",
       "3229       0.0            1              0.0     0  \n",
       "3230       0.0            1              0.0     0  \n",
       "3231       0.0            0              NaN     0  \n",
       "3232       0.0            1              0.0     0  \n",
       "3233       0.0            0              NaN     0  \n",
       "\n",
       "[5084 rows x 8 columns]"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_qs = merged[['model','question', 'solution', 'answer', 'accuracy', 'solvability', 'appropriateness', 'good']]\n",
    "mathwell_annotated = pd.read_csv('data/mathwell_annotations.csv')\n",
    "def recode(df):\n",
    "    if df['solvability']==0:\n",
    "        return np.nan\n",
    "    else:\n",
    "        return df['appropriateness']\n",
    "mathwell_annotated['appropriateness']= mathwell_annotated.apply(recode, axis=1)\n",
    "all_qs = pd.concat([all_qs, mathwell_annotated])\n",
    "replace_map = {np.nan: 'finetuned_llama'}\n",
    "all_qs['model'] = all_qs['model'].replace(replace_map)\n",
    "all_qs = all_qs.drop(['Unnamed: 0'], axis=1)\n",
    "mathwell_an = all_qs[all_qs['model']!='llama']\n",
    "mathwell_an = mathwell_an[mathwell_an['model']!='llema']\n",
    "mathwell_an = mathwell_an[mathwell_an['model']!='mammoth']\n",
    "mathwell_an = mathwell_an[mathwell_an['model']!='gpt35']\n",
    "mathwell_an = mathwell_an[mathwell_an['model']!='gpt4']\n",
    "mathwell_an = mathwell_an[mathwell_an['model']!='llama_gsm']\n",
    "mathwell_an.to_csv('data/mathwell_annotations_final.csv', index = False)\n",
    "all_qs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "42c09784-3aa7-467e-9244-b6234edbd100",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>solvability</th>\n",
       "      <th>appropriateness</th>\n",
       "      <th>good</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mathwell</td>\n",
       "      <td>LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...</td>\n",
       "      <td>def solution():\\n    #LeBron James is 6'9\" tal...</td>\n",
       "      <td>205.74</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mathwell</td>\n",
       "      <td>Captain America has 1200 ryo. He spends 400 ry...</td>\n",
       "      <td>def solution():\\n    #Captain America started ...</td>\n",
       "      <td>300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mathwell</td>\n",
       "      <td>A 250-car train is 2.5 miles long. How many 25...</td>\n",
       "      <td>def solution():\\n    #1 train is 2.5 miles lon...</td>\n",
       "      <td>211200.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mathwell</td>\n",
       "      <td>3 scoops of ice cream cost $4. \\nHow much does...</td>\n",
       "      <td>def solution():\\n    #3 scoops of ice cream co...</td>\n",
       "      <td>13.333333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mathwell</td>\n",
       "      <td>Iron Man has 500 gold coins. He can use 10 gol...</td>\n",
       "      <td>def solution():\\n    #Iron Man has 500 gold co...</td>\n",
       "      <td>16.666667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3229</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>A field hockey game has 11 players per team. T...</td>\n",
       "      <td>def solution():\\n    #A field hockey game has ...</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>100 turtles are at the pond. 10 turtles are ea...</td>\n",
       "      <td>def solution():\\n    #Total turtles\\n    total...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3231</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>Barbie has 1200 Instagram followers. 200 more ...</td>\n",
       "      <td>def solution():\\n    #Barbie started with 1200...</td>\n",
       "      <td>1150</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3232</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>1234567890 is a 10-digit number. \\nHow many 3-...</td>\n",
       "      <td>def solution():\\n    #10-digit number has 10 d...</td>\n",
       "      <td>3.3333333333333335</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3233</th>\n",
       "      <td>finetuned_llama</td>\n",
       "      <td>7 cats and 23 dogs are in a cage. 14 of the an...</td>\n",
       "      <td>def solution():\\n    #Total cats and dogs\\n   ...</td>\n",
       "      <td>13</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3734 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                model                                           question  \\\n",
       "0            mathwell  LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...   \n",
       "1            mathwell  Captain America has 1200 ryo. He spends 400 ry...   \n",
       "2            mathwell  A 250-car train is 2.5 miles long. How many 25...   \n",
       "3            mathwell  3 scoops of ice cream cost $4. \\nHow much does...   \n",
       "4            mathwell  Iron Man has 500 gold coins. He can use 10 gol...   \n",
       "...               ...                                                ...   \n",
       "3229  finetuned_llama  A field hockey game has 11 players per team. T...   \n",
       "3230  finetuned_llama  100 turtles are at the pond. 10 turtles are ea...   \n",
       "3231  finetuned_llama  Barbie has 1200 Instagram followers. 200 more ...   \n",
       "3232  finetuned_llama  1234567890 is a 10-digit number. \\nHow many 3-...   \n",
       "3233  finetuned_llama  7 cats and 23 dogs are in a cage. 14 of the an...   \n",
       "\n",
       "                                               solution              answer  \\\n",
       "0     def solution():\\n    #LeBron James is 6'9\" tal...              205.74   \n",
       "1     def solution():\\n    #Captain America started ...               300.0   \n",
       "2     def solution():\\n    #1 train is 2.5 miles lon...            211200.0   \n",
       "3     def solution():\\n    #3 scoops of ice cream co...           13.333333   \n",
       "4     def solution():\\n    #Iron Man has 500 gold co...           16.666667   \n",
       "...                                                 ...                 ...   \n",
       "3229  def solution():\\n    #A field hockey game has ...                  33   \n",
       "3230  def solution():\\n    #Total turtles\\n    total...                   0   \n",
       "3231  def solution():\\n    #Barbie started with 1200...                1150   \n",
       "3232  def solution():\\n    #10-digit number has 10 d...  3.3333333333333335   \n",
       "3233  def solution():\\n    #Total cats and dogs\\n   ...                  13   \n",
       "\n",
       "      accuracy  solvability  appropriateness  good  \n",
       "0          1.0            1              1.0     1  \n",
       "1          1.0            1              1.0     1  \n",
       "2          0.0            1              1.0     0  \n",
       "3          1.0            1              1.0     1  \n",
       "4          1.0            1              0.0     0  \n",
       "...        ...          ...              ...   ...  \n",
       "3229       0.0            1              0.0     0  \n",
       "3230       0.0            1              0.0     0  \n",
       "3231       0.0            0              NaN     0  \n",
       "3232       0.0            1              0.0     0  \n",
       "3233       0.0            0              NaN     0  \n",
       "\n",
       "[3734 rows x 8 columns]"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mathwell_an"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "1bab06c0-cd31-4d72-a76c-bad586d0dafd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall num questions: 5084\n",
      "Overall percentage appropriate: 0.7807563427477262 Standard Deviation: 0.4137340643540501 Standard Error: 0.6401607486187453\n",
      "Overall percentage solvable: 0.8217938630999213 Standard Deviation: 0.38268617648306696 Standard Error: 0.5367631834172857\n",
      "Overall percentage accurate: 0.8728150873965042 Standard Deviation: 0.33317999731306946 Standard Error: 0.5053404936423489\n",
      "Overall percentage good in all metrics: 0.5841856805664831 Standard Deviation: 0.4928618175387074 Standard Error: 0.6912977118696937\n",
      "Number of good questions: 2970\n"
     ]
    }
   ],
   "source": [
    "print(f'''Overall num questions: {len(all_qs)}\n",
    "Overall percentage appropriate: {all_qs['appropriateness'].mean()} Standard Deviation: {np.std(all_qs['appropriateness'])} Standard Error: {st.sem(all_qs['appropriateness'].dropna())*100}\n",
    "Overall percentage solvable: {all_qs['solvability'].mean()} Standard Deviation: {np.std(all_qs['solvability'])} Standard Error: {st.sem(all_qs['solvability'])*100}\n",
    "Overall percentage accurate: {all_qs['accuracy'].mean()} Standard Deviation: {np.std(all_qs['accuracy'])} Standard Error: {st.sem(all_qs['accuracy'].dropna())*100}\n",
    "Overall percentage good in all metrics: {all_qs['good'].mean()} Standard Deviation: {np.std(all_qs['good'])} Standard Error: {st.sem(all_qs['good'])*100}\n",
    "Number of good questions: {len(all_qs[all_qs['good']==1])}''')\n",
    "all_qs = all_qs.rename(columns = {'good': 'mac'})\n",
    "all_qs.to_csv('data/annotations.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1104f95b-ec50-40e1-b7a5-77d565f9b4bc",
   "metadata": {},
   "source": [
    "## Format Good Data in Instruction Format for Further Finetuning for Question Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "d675956a-0741-444a-9527-6afe7a37758b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>solution</th>\n",
       "      <th>answer</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...</td>\n",
       "      <td>def solution():\\n    #LeBron James is 6'9\" tal...</td>\n",
       "      <td>205.74</td>\n",
       "      <td>basketball</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Captain America has 1200 ryo. He spends 400 ry...</td>\n",
       "      <td>def solution():\\n    #Captain America started ...</td>\n",
       "      <td>300.0</td>\n",
       "      <td>Captain America</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3 scoops of ice cream cost $4. \\nHow much does...</td>\n",
       "      <td>def solution():\\n    #3 scoops of ice cream co...</td>\n",
       "      <td>13.333333</td>\n",
       "      <td>ice cream</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>A volleyball team is 6 players. \\n3 of the pla...</td>\n",
       "      <td>def solution():\\n    #3 of the players are 16 ...</td>\n",
       "      <td>16.666667</td>\n",
       "      <td>volleyball</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Superman can fly at a speed of 1,000 miles per...</td>\n",
       "      <td>def solution():\\n    #Superman can fly at a sp...</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Superman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1901</th>\n",
       "      <td>30 unicorns are in a herd. 10 unicorns in the ...</td>\n",
       "      <td>def solution():\\n    #Total number of unicorns...</td>\n",
       "      <td>17</td>\n",
       "      <td>unicorns</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1902</th>\n",
       "      <td>A pizza has 8 slices. 4 of the slices have pep...</td>\n",
       "      <td>def solution():\\n    #A pizza has 8 slices\\n  ...</td>\n",
       "      <td>4</td>\n",
       "      <td>pizza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1903</th>\n",
       "      <td>12000 people were at a rally. 10000 of them we...</td>\n",
       "      <td>def solution():\\n    #Total number of people a...</td>\n",
       "      <td>500</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1904</th>\n",
       "      <td>A bakery made 300 muffins. 150 of the muffins ...</td>\n",
       "      <td>def solution():\\n    #Total muffins\\n    total...</td>\n",
       "      <td>60</td>\n",
       "      <td>unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1905</th>\n",
       "      <td>A pizza is cut into 12 slices. 5 slices are to...</td>\n",
       "      <td>def solution():\\n    #A pizza is cut into 12 s...</td>\n",
       "      <td>6</td>\n",
       "      <td>pizza</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2093 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question  \\\n",
       "0     LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...   \n",
       "1     Captain America has 1200 ryo. He spends 400 ry...   \n",
       "3     3 scoops of ice cream cost $4. \\nHow much does...   \n",
       "5     A volleyball team is 6 players. \\n3 of the pla...   \n",
       "6     Superman can fly at a speed of 1,000 miles per...   \n",
       "...                                                 ...   \n",
       "1901  30 unicorns are in a herd. 10 unicorns in the ...   \n",
       "1902  A pizza has 8 slices. 4 of the slices have pep...   \n",
       "1903  12000 people were at a rally. 10000 of them we...   \n",
       "1904  A bakery made 300 muffins. 150 of the muffins ...   \n",
       "1905  A pizza is cut into 12 slices. 5 slices are to...   \n",
       "\n",
       "                                               solution     answer  \\\n",
       "0     def solution():\\n    #LeBron James is 6'9\" tal...     205.74   \n",
       "1     def solution():\\n    #Captain America started ...      300.0   \n",
       "3     def solution():\\n    #3 scoops of ice cream co...  13.333333   \n",
       "5     def solution():\\n    #3 of the players are 16 ...  16.666667   \n",
       "6     def solution():\\n    #Superman can fly at a sp...        0.2   \n",
       "...                                                 ...        ...   \n",
       "1901  def solution():\\n    #Total number of unicorns...         17   \n",
       "1902  def solution():\\n    #A pizza has 8 slices\\n  ...          4   \n",
       "1903  def solution():\\n    #Total number of people a...        500   \n",
       "1904  def solution():\\n    #Total muffins\\n    total...         60   \n",
       "1905  def solution():\\n    #A pizza is cut into 12 s...          6   \n",
       "\n",
       "                topic  \n",
       "0          basketball  \n",
       "1     Captain America  \n",
       "3           ice cream  \n",
       "5          volleyball  \n",
       "6            Superman  \n",
       "...               ...  \n",
       "1901         unicorns  \n",
       "1902            pizza  \n",
       "1903      unspecified  \n",
       "1904      unspecified  \n",
       "1905            pizza  \n",
       "\n",
       "[2093 rows x 4 columns]"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged=merged.query(\"good==1 & model=='mathwell'\")\n",
    "sgsm_train = merged[['question', 'solution', 'answer', 'topic_y']]\n",
    "sgsm_train = sgsm_train.rename(columns = {'topic_y': 'topic'})\n",
    "train = pd.read_csv('data/finetuning_data.csv')\n",
    "sgsm_train = pd.concat([sgsm_train, train])\n",
    "replace_map = {np.nan: 'unspecified'}\n",
    "sgsm_train['topic'] = sgsm_train['topic'].replace(replace_map)\n",
    "sgsm_train = sgsm_train.drop(['Unnamed: 0'], axis=1)\n",
    "def parse_solution(df):\n",
    "    if \"\\nBe\" in df['solution']:\n",
    "        solution = df['solution'].split(\"\\nBe\")[0]\n",
    "    if \"##\" in df['solution']:\n",
    "        solution = df['solution'].split(\"##\")[0]\n",
    "    if \"print(solution())\" in df['solution']:\n",
    "        solution = df['solution'].split(\"print(solution())\")[0]\n",
    "    else:\n",
    "        solution = df['solution']\n",
    "    return solution\n",
    "sgsm_train['solution'] = sgsm_train.apply(parse_solution, axis=1)\n",
    "sgsm_train.to_csv('data/sgsm_train.csv')\n",
    "sgsm_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "0746d7aa-2fa4-42ee-b35a-6d3b4ab76f05",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/2997887767.py:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  merged['solution'] = merged.apply(parse_solution, axis = 1)\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/2997887767.py:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  merged['output'] = \"Question: \" + merged['question'] + \"\\n\" + \"Solution:\\n\" + merged['solution']\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/2997887767.py:14: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  merged['instruction'] = \"Write a grade school math word problem and Python function with a commented out step-by-step solution to solve the word problem.\"\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/2997887767.py:15: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  merged['input']=''\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>output</th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Question: LeBron James is 6'9\" tall. 1 inch is...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Question: Captain America has 1200 ryo. He spe...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Question: 3 scoops of ice cream cost $4. \\nHow...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Question: A volleyball team is 6 players. \\n3 ...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Question: Superman can fly at a speed of 1,000...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>Question: A candy store has 120 lollipops. 30 ...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>Question: Wonder Woman has 100 lassos. She las...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>Question: Superman can punch 2000 times per mi...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>Question: A 25-meter pool is 2.5 times as long...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>Question: Serena Williams has 24 Grand Slam ti...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>187 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                output  \\\n",
       "0    Question: LeBron James is 6'9\" tall. 1 inch is...   \n",
       "1    Question: Captain America has 1200 ryo. He spe...   \n",
       "3    Question: 3 scoops of ice cream cost $4. \\nHow...   \n",
       "5    Question: A volleyball team is 6 players. \\n3 ...   \n",
       "6    Question: Superman can fly at a speed of 1,000...   \n",
       "..                                                 ...   \n",
       "218  Question: A candy store has 120 lollipops. 30 ...   \n",
       "219  Question: Wonder Woman has 100 lassos. She las...   \n",
       "220  Question: Superman can punch 2000 times per mi...   \n",
       "221  Question: A 25-meter pool is 2.5 times as long...   \n",
       "231  Question: Serena Williams has 24 Grand Slam ti...   \n",
       "\n",
       "                                           instruction input  \n",
       "0    Write a grade school math word problem and Pyt...        \n",
       "1    Write a grade school math word problem and Pyt...        \n",
       "3    Write a grade school math word problem and Pyt...        \n",
       "5    Write a grade school math word problem and Pyt...        \n",
       "6    Write a grade school math word problem and Pyt...        \n",
       "..                                                 ...   ...  \n",
       "218  Write a grade school math word problem and Pyt...        \n",
       "219  Write a grade school math word problem and Pyt...        \n",
       "220  Write a grade school math word problem and Pyt...        \n",
       "221  Write a grade school math word problem and Pyt...        \n",
       "231  Write a grade school math word problem and Pyt...        \n",
       "\n",
       "[187 rows x 3 columns]"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_copy = merged\n",
    "def parse_solution(df):\n",
    "    if \"\\nBe\" in df['solution']:\n",
    "        solution = df['solution'].split(\"\\nBe\")[0]\n",
    "    if \"##\" in df['solution']:\n",
    "        solution = df['solution'].split(\"##\")[0]\n",
    "    if \"print(solution())\" in df['solution']:\n",
    "        solution = df['solution'].split(\"print(solution())\")[0]\n",
    "    else:\n",
    "        solution = df['solution']\n",
    "    return solution\n",
    "merged['solution'] = merged.apply(parse_solution, axis = 1)\n",
    "merged['output'] = \"Question: \" + merged['question'] + \"\\n\" + \"Solution:\\n\" + merged['solution']\n",
    "merged['instruction'] = \"Write a grade school math word problem and Python function with a commented out step-by-step solution to solve the word problem.\"\n",
    "merged['input']=''\n",
    "merged = merged[['output', 'instruction', 'input']]\n",
    "merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "2d597a98-9224-4d9a-84f7-08d0b0fe8d99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>output</th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Question: LeBron James has 30000 points. He ha...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Question: A class has 120 students. 40 of them...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Question: Naruto has 5000 ryo. He spends 1000 ...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.0</td>\n",
       "      <td>Question: Hello Kitty has 12 pencils. She boug...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.0</td>\n",
       "      <td>Question: Wonder Woman's lasso of truth can ho...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Question: A candy store has 120 lollipops. 30 ...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Question: Wonder Woman has 100 lassos. She las...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Question: Superman can punch 2000 times per mi...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Question: A 25-meter pool is 2.5 times as long...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Question: Serena Williams has 24 Grand Slam ti...</td>\n",
       "      <td>Write a grade school math word problem and Pyt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2093 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0                                             output  \\\n",
       "0           0.0  Question: LeBron James has 30000 points. He ha...   \n",
       "1           1.0  Question: A class has 120 students. 40 of them...   \n",
       "2           5.0  Question: Naruto has 5000 ryo. He spends 1000 ...   \n",
       "3           6.0  Question: Hello Kitty has 12 pencils. She boug...   \n",
       "4           7.0  Question: Wonder Woman's lasso of truth can ho...   \n",
       "..          ...                                                ...   \n",
       "218         NaN  Question: A candy store has 120 lollipops. 30 ...   \n",
       "219         NaN  Question: Wonder Woman has 100 lassos. She las...   \n",
       "220         NaN  Question: Superman can punch 2000 times per mi...   \n",
       "221         NaN  Question: A 25-meter pool is 2.5 times as long...   \n",
       "231         NaN  Question: Serena Williams has 24 Grand Slam ti...   \n",
       "\n",
       "                                           instruction input  \n",
       "0    Write a grade school math word problem and Pyt...        \n",
       "1    Write a grade school math word problem and Pyt...        \n",
       "2    Write a grade school math word problem and Pyt...        \n",
       "3    Write a grade school math word problem and Pyt...        \n",
       "4    Write a grade school math word problem and Pyt...        \n",
       "..                                                 ...   ...  \n",
       "218  Write a grade school math word problem and Pyt...        \n",
       "219  Write a grade school math word problem and Pyt...        \n",
       "220  Write a grade school math word problem and Pyt...        \n",
       "221  Write a grade school math word problem and Pyt...        \n",
       "231  Write a grade school math word problem and Pyt...        \n",
       "\n",
       "[2093 rows x 4 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save DataFrame to a JSON file\n",
    "mathwell = pd.read_csv('data/mathwell.csv')\n",
    "mathwell = pd.concat([mathwell, merged])\n",
    "replace_map = {np.nan: \"\"}\n",
    "mathwell['input'] = mathwell['input'].replace(replace_map)\n",
    "mathwell.to_csv('data/mathwell_final.csv')\n",
    "mathwell.to_json('data/mathwell_final.json', orient='records', lines=True)  # Saves each row as a separate JSON object in the file\n",
    "import json\n",
    "\n",
    "file_path = \"data/mathwell_final.json\"\n",
    "data = []\n",
    "\n",
    "with open(file_path, 'r') as json_file:\n",
    "    for line in json_file:\n",
    "        try:\n",
    "            obj = json.loads(line)\n",
    "            data.append(obj)\n",
    "        except json.JSONDecodeError as e:\n",
    "            print(f\"Error parsing JSON on line: {line}\")\n",
    "output_file = \"data/mathwell_final.json\"\n",
    "with open(output_file, 'w') as json_file:\n",
    "    json.dump(data, json_file)\n",
    "\n",
    "mathwell"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1020c94c-9f6a-4fd5-b343-78513fb50be8",
   "metadata": {},
   "source": [
    "## Format Good Data in Instruction Format for Further Finetuning for Question Answering for MAmmoTH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "03aff65d-f3ad-41bc-9f60-da9a2ab84dff",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3035772303.py:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  merged_copy['solution'] = merged_copy.apply(parse_solution, axis = 1)\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3035772303.py:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  merged_copy['output'] = merged_copy['solution']\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3035772303.py:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  merged_copy['instruction'] = merged_copy['question'] + \" Let's write a program.\"\n",
      "/var/folders/ms/lv6zlm_16wx8dzf7c_t51b0r0000gn/T/ipykernel_94037/3035772303.py:14: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  merged_copy['input']=''\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>output</th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>def solution():\\n    #LeBron James is 6'9\" tal...</td>\n",
       "      <td>LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>def solution():\\n    #Captain America started ...</td>\n",
       "      <td>Captain America has 1200 ryo. He spends 400 ry...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>def solution():\\n    #3 scoops of ice cream co...</td>\n",
       "      <td>3 scoops of ice cream cost $4. \\nHow much does...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>def solution():\\n    #3 of the players are 16 ...</td>\n",
       "      <td>A volleyball team is 6 players. \\n3 of the pla...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>def solution():\\n    #Superman can fly at a sp...</td>\n",
       "      <td>Superman can fly at a speed of 1,000 miles per...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>def solution():\\n    #Total lollipops\\n    tot...</td>\n",
       "      <td>A candy store has 120 lollipops. 30 of the lol...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>def solution():\\n    #Wonder Woman started wit...</td>\n",
       "      <td>Wonder Woman has 100 lassos. She lassos 20 bad...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>def solution():\\n    #Superman can punch 2000 ...</td>\n",
       "      <td>Superman can punch 2000 times per minute.  How...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>def solution():\\n    #A 25-meter pool is 2.5 t...</td>\n",
       "      <td>A 25-meter pool is 2.5 times as long as a 10-m...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>def solution():\\n    #Serena Williams has 24 G...</td>\n",
       "      <td>Serena Williams has 24 Grand Slam titles. 13 o...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>187 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                output  \\\n",
       "0    def solution():\\n    #LeBron James is 6'9\" tal...   \n",
       "1    def solution():\\n    #Captain America started ...   \n",
       "3    def solution():\\n    #3 scoops of ice cream co...   \n",
       "5    def solution():\\n    #3 of the players are 16 ...   \n",
       "6    def solution():\\n    #Superman can fly at a sp...   \n",
       "..                                                 ...   \n",
       "218  def solution():\\n    #Total lollipops\\n    tot...   \n",
       "219  def solution():\\n    #Wonder Woman started wit...   \n",
       "220  def solution():\\n    #Superman can punch 2000 ...   \n",
       "221  def solution():\\n    #A 25-meter pool is 2.5 t...   \n",
       "231  def solution():\\n    #Serena Williams has 24 G...   \n",
       "\n",
       "                                           instruction input  \n",
       "0    LeBron James is 6'9\" tall. 1 inch is 2.54 cm. ...        \n",
       "1    Captain America has 1200 ryo. He spends 400 ry...        \n",
       "3    3 scoops of ice cream cost $4. \\nHow much does...        \n",
       "5    A volleyball team is 6 players. \\n3 of the pla...        \n",
       "6    Superman can fly at a speed of 1,000 miles per...        \n",
       "..                                                 ...   ...  \n",
       "218  A candy store has 120 lollipops. 30 of the lol...        \n",
       "219  Wonder Woman has 100 lassos. She lassos 20 bad...        \n",
       "220  Superman can punch 2000 times per minute.  How...        \n",
       "221  A 25-meter pool is 2.5 times as long as a 10-m...        \n",
       "231  Serena Williams has 24 Grand Slam titles. 13 o...        \n",
       "\n",
       "[187 rows x 3 columns]"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def parse_solution(df):\n",
    "    if \"\\nBe\" in df['solution']:\n",
    "        solution = df['solution'].split(\"\\nBe\")[0]\n",
    "    if \"##\" in df['solution']:\n",
    "        solution = df['solution'].split(\"##\")[0]\n",
    "    if \"print(solution())\" in df['solution']:\n",
    "        solution = df['solution'].split(\"print(solution())\")[0]\n",
    "    else:\n",
    "        solution = df['solution']\n",
    "    return solution\n",
    "merged_copy['solution'] = merged_copy.apply(parse_solution, axis = 1)\n",
    "merged_copy['output'] = merged_copy['solution']\n",
    "merged_copy['instruction'] = merged_copy['question'] + \" Let's write a program.\"\n",
    "merged_copy['input']=''\n",
    "merged_copy = merged_copy[['output', 'instruction', 'input']]\n",
    "merged_copy.drop_duplicates(subset = 'instruction')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "0548645c-4c20-4f69-a2d8-70ccac1756c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save DataFrame to a JSON file\n",
    "mathwell = pd.read_csv('data/mathwell_qa.csv')\n",
    "mathwell = pd.concat([mathwell, merged_copy])\n",
    "mathwell['input']=\"\"\n",
    "mathwell = mathwell[['output', 'instruction', 'input']]\n",
    "mathwell.to_csv('data/mathwell_final_qa.csv')\n",
    "mathwell.to_json('data/mathwell_final_qa.json', orient='records', lines=True)  # Saves each row as a separate JSON object in the file\n",
    "import json\n",
    "\n",
    "file_path = \"data/mathwell_final_qa.json\"\n",
    "data = []\n",
    "\n",
    "with open(file_path, 'r') as json_file:\n",
    "    for line in json_file:\n",
    "        try:\n",
    "            obj = json.loads(line)\n",
    "            data.append(obj)\n",
    "        except json.JSONDecodeError as e:\n",
    "            print(f\"Error parsing JSON on line: {line}\")\n",
    "output_file = \"data/mathwell_final_qa.json\"\n",
    "with open(output_file, 'w') as json_file:\n",
    "    json.dump(data, json_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "929499bf-9a36-4d3e-91a4-13b6829cf317",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}