LLM_zero_shot_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\tighu\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python312\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "C:\\Users\\tighu\\AppData\\Local\\Temp\\ipykernel_23364\\183117859.py:2: DeprecationWarning: \n",
      "Pyarrow will become a required dependency of pandas in the next major release of pandas (pandas 3.0),\n",
      "(to allow more performant data types, such as the Arrow string type, and better interoperability with other libraries)\n",
      "but was not found to be installed on your system.\n",
      "If this would cause problems for you,\n",
      "please provide us feedback at https://github.com/pandas-dev/pandas/issues/54466\n",
      "        \n",
      "  import pandas as pd\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = pipeline(\"zero-shot-classification\",\n",
    "                      model=\"facebook/bart-large-mnli\", device= 0)\n",
    "movie_reviews_df = pd.read_csv(\"IMDB Dataset.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 2)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_reviews_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n",
      "26000\n",
      "27000\n",
      "28000\n",
      "29000\n",
      "30000\n",
      "31000\n",
      "32000\n",
      "33000\n",
      "34000\n",
      "35000\n",
      "36000\n",
      "37000\n",
      "38000\n",
      "39000\n",
      "40000\n",
      "41000\n",
      "42000\n",
      "43000\n",
      "44000\n",
      "45000\n",
      "46000\n",
      "47000\n",
      "48000\n",
      "49000\n",
      "50000\n"
     ]
    }
   ],
   "source": [
    "# use zero shot classifier for the three labels\n",
    "candidate_labels = ['positive', 'negative', 'neutral']\n",
    "\n",
    "llm_label = []\n",
    "llm_confidence = []\n",
    "count = 0\n",
    "for review in movie_reviews_df['review']:\n",
    "    \n",
    "    zero_shot_result = classifier(review, candidate_labels)\n",
    "    \n",
    "    llm_label.append(zero_shot_result['labels'][0])\n",
    "    llm_confidence.append(zero_shot_result['scores'][0])  \n",
    "\n",
    "    count +=1\n",
    "\n",
    "    if count % 1000 == 0: print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_reviews_df[\"llm_label_bert\"] = llm_label\n",
    "movie_reviews_df[\"llm_confidence_bert\"] = llm_confidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>llm_label_bert</th>\n",
       "      <th>llm_confidence_bert</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>One of the other reviewers has mentioned that ...</td>\n",
       "      <td>positive</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.410353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n",
       "      <td>positive</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.948007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I thought this was a wonderful way to spend ti...</td>\n",
       "      <td>positive</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.933053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Basically there's a family where a little boy ...</td>\n",
       "      <td>negative</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.950203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n",
       "      <td>positive</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.768185</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment llm_label_bert  \\\n",
       "0  One of the other reviewers has mentioned that ...  positive       positive   \n",
       "1  A wonderful little production. <br /><br />The...  positive       positive   \n",
       "2  I thought this was a wonderful way to spend ti...  positive       positive   \n",
       "3  Basically there's a family where a little boy ...  negative       negative   \n",
       "4  Petter Mattei's \"Love in the Time of Money\" is...  positive       positive   \n",
       "\n",
       "   llm_confidence_bert  \n",
       "0             0.410353  \n",
       "1             0.948007  \n",
       "2             0.933053  \n",
       "3             0.950203  \n",
       "4             0.768185  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_reviews_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sentiment\n",
       "positive    25000\n",
       "negative    25000\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_reviews_df[\"sentiment\"].value_counts()\n",
    "#even split to begin with initial dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "llm_label_bert\n",
       "negative    27149\n",
       "positive    22610\n",
       "neutral       241\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split based on BERT LLM\n",
    "# introduction of minority class neutral 250:  0.5%\n",
    "movie_reviews_df[\"llm_label_bert\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all the rows where two are unequal\n",
    "# it accounts for around 12% of the entire dataset\n",
    "label_errors_df = movie_reviews_df.query('llm_label_bert != sentiment')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>llm_label_bert</th>\n",
       "      <th>llm_confidence_bert</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Some films just simply should not be remade. T...</td>\n",
       "      <td>positive</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.811583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>The Karen Carpenter Story shows a little more ...</td>\n",
       "      <td>positive</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.761770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>'War movie' is a Hollywood genre that has been...</td>\n",
       "      <td>positive</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.601619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Taut and organically gripping, Edward Dmytryk'...</td>\n",
       "      <td>positive</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.753970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>\"Ardh Satya\" is one of the finest film ever ma...</td>\n",
       "      <td>positive</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.636056</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               review sentiment  \\\n",
       "16  Some films just simply should not be remade. T...  positive   \n",
       "25  The Karen Carpenter Story shows a little more ...  positive   \n",
       "29  'War movie' is a Hollywood genre that has been...  positive   \n",
       "30  Taut and organically gripping, Edward Dmytryk'...  positive   \n",
       "31  \"Ardh Satya\" is one of the finest film ever ma...  positive   \n",
       "\n",
       "   llm_label_bert  llm_confidence_bert  \n",
       "16       negative             0.811583  \n",
       "25       negative             0.761770  \n",
       "29       negative             0.601619  \n",
       "30       negative             0.753970  \n",
       "31       negative             0.636056  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# putting some of the cases where a new sentiment was assigned to the movie rewiew\n",
    "\n",
    "label_errors_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Some films just simply should not be remade. This is one of them. In and of itself it is not a bad film. But it fails to capture the flavor and the terror of the 1963 film of the same title. Liam Neeson was excellent as he always is, and most of the cast holds up, with the exception of Owen Wilson, who just did not bring the right feel to the character of Luke. But the major fault with this version is that it strayed too far from the Shirley Jackson story in it's attempts to be grandiose and lost some of the thrill of the earlier film in a trade off for snazzier special effects. Again I will say that in and of itself it is not a bad film. But you will enjoy the friction of terror in the older version much more.\n"
     ]
    }
   ],
   "source": [
    "label_errors_df['review'][16]\n",
    "\n",
    "# Some films just simply should not be remade. This is one of them. In and of itself it is not a bad film. But it fails to capture the flavor and \n",
    "# the terror of the 1963 film of the same title. Liam Neeson was excellent as he always is, and most of the cast holds up, with the exception of Owen Wilson, \n",
    "# who just did not bring the right feel to the character of Luke. But the major fault with this version is that it strayed too far from the Shirley Jackson \n",
    "# story in it's attempts to be grandiose and lost some of the thrill of the earlier film in a trade off for snazzier special effects. Again I will say that \n",
    "# in and of itself it is not a bad film. But you will enjoy the friction of terror in the older version much more."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"The Karen Carpenter Story shows a little more about singer Karen Carpenter's complex life. Though it fails in giving accurate facts, and details.<br /><br />Cynthia Gibb (portrays Karen) was not a fine election. She is a good actress , but plays a very naive and sort of dumb Karen Carpenter. I think that the role needed a stronger character. Someone with a stronger personality.<br /><br />Louise Fletcher role as Agnes Carpenter is terrific, she does a great job as Karen's mother.<br /><br />It has great songs, which could have been included in a soundtrack album. Unfortunately they weren't, though this movie was on the top of the ratings in USA and other several countries\""
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_errors_df['review'][25]\n",
    "# \"The Karen Carpenter Story shows a little more about singer Karen Carpenter's complex life. Though it fails in giving accurate facts, and details.<br /><br \n",
    "# />Cynthia Gibb (portrays Karen) was not a fine election. She is a good actress , but plays a very naive and sort of dumb Karen Carpenter. I think that the \n",
    "# role needed a stronger character. Someone with a stronger personality.<br /><br />Louise Fletcher role as Agnes Carpenter is terrific, she does a great job \n",
    "# as Karen's mother.<br /><br />It has great songs, which could have been included in a soundtrack album. Unfortunately they weren't, though this movie was on \n",
    "# the top of the ratings in USA and other several countries\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Although both reviews were negatively leaning they were initially assigned a positive label in the dataset. Number of such rewviews in the dataset amount\n",
    "# to 6K which is about 12% of the entire dataset. So any model trained with the original dataset would have skewed trained and test set which would\n",
    "# severly impact its working in the real world. In this excercise zero shot classifier in form of Bert Large is used to improve accuracy of dataset by 12% and\n",
    "# adding more diversity to reviews labels by adding a neutral label."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}