simple_baseline_logistic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\taow\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\taow\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>par_id</th>\n",
       "      <th>art_id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>country</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>@@24942188</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>ph</td>\n",
       "      <td>We 're living in times of absolute insanity , ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>@@21968160</td>\n",
       "      <td>migrant</td>\n",
       "      <td>gh</td>\n",
       "      <td>In Libya today , there are countless number of...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>@@16584954</td>\n",
       "      <td>immigrant</td>\n",
       "      <td>ie</td>\n",
       "      <td>White House press secretary Sean Spicer said t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>@@7811231</td>\n",
       "      <td>disabled</td>\n",
       "      <td>nz</td>\n",
       "      <td>Council customers only signs would be displaye...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>@@1494111</td>\n",
       "      <td>refugee</td>\n",
       "      <td>ca</td>\n",
       "      <td>\" Just like we received migrants fleeing El Sa...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10464</th>\n",
       "      <td>10465</td>\n",
       "      <td>@@14297363</td>\n",
       "      <td>women</td>\n",
       "      <td>lk</td>\n",
       "      <td>Sri Lankan norms and culture inhibit women fro...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10465</th>\n",
       "      <td>10466</td>\n",
       "      <td>@@70091353</td>\n",
       "      <td>vulnerable</td>\n",
       "      <td>ph</td>\n",
       "      <td>He added that the AFP will continue to bank on...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10466</th>\n",
       "      <td>10467</td>\n",
       "      <td>@@20282330</td>\n",
       "      <td>in-need</td>\n",
       "      <td>ng</td>\n",
       "      <td>\" She has one huge platform , and information ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10467</th>\n",
       "      <td>10468</td>\n",
       "      <td>@@16753236</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>in</td>\n",
       "      <td>\" Anja Ringgren Loven I ca n't find a word to ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10468</th>\n",
       "      <td>10469</td>\n",
       "      <td>@@16779383</td>\n",
       "      <td>homeless</td>\n",
       "      <td>ie</td>\n",
       "      <td>\" Guinness World Record of 540lbs of 7-layer m...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10469 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       par_id      art_id     keyword country  \\\n",
       "0           1  @@24942188    hopeless      ph   \n",
       "1           2  @@21968160     migrant      gh   \n",
       "2           3  @@16584954   immigrant      ie   \n",
       "3           4   @@7811231    disabled      nz   \n",
       "4           5   @@1494111     refugee      ca   \n",
       "...       ...         ...         ...     ...   \n",
       "10464   10465  @@14297363       women      lk   \n",
       "10465   10466  @@70091353  vulnerable      ph   \n",
       "10466   10467  @@20282330     in-need      ng   \n",
       "10467   10468  @@16753236    hopeless      in   \n",
       "10468   10469  @@16779383    homeless      ie   \n",
       "\n",
       "                                                    text  label  \n",
       "0      We 're living in times of absolute insanity , ...      0  \n",
       "1      In Libya today , there are countless number of...      0  \n",
       "2      White House press secretary Sean Spicer said t...      0  \n",
       "3      Council customers only signs would be displaye...      0  \n",
       "4      \" Just like we received migrants fleeing El Sa...      0  \n",
       "...                                                  ...    ...  \n",
       "10464  Sri Lankan norms and culture inhibit women fro...      0  \n",
       "10465  He added that the AFP will continue to bank on...      0  \n",
       "10466  \" She has one huge platform , and information ...      1  \n",
       "10467  \" Anja Ringgren Loven I ca n't find a word to ...      1  \n",
       "10468  \" Guinness World Record of 540lbs of 7-layer m...      1  \n",
       "\n",
       "[10469 rows x 6 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import torch\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import f1_score\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "\n",
    "# Download necessary NLTK data\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "\n",
    "cuda_available = torch.cuda.is_available()\n",
    "device = 'cuda:0' if cuda_available else 'cpu'\n",
    "\n",
    "torch.manual_seed(0)\n",
    "\n",
    "data = pd.read_csv(\"data/dontpatronizeme_pcl.tsv\",\n",
    "                       sep=\"\\t\",\n",
    "                       names=['par_id', 'art_id', 'keyword', 'country', 'text', 'label'],\n",
    "                       skiprows=4)\n",
    "\n",
    "data['label'] = data['label'].apply(lambda x: 0 if x in [0, 1] else 1)\n",
    "\n",
    "trids = pd.read_csv('data/train_semeval_parids-labels.csv')\n",
    "teids = pd.read_csv('data/dev_semeval_parids-labels.csv')\n",
    "\n",
    "trids.par_id = trids.par_id\n",
    "teids.par_id = teids.par_id\n",
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rebuild training set\n",
    "\n",
    "rows = [] # will contain par_id, label and text\n",
    "for idx in range(len(trids)):\n",
    "  parid = trids.par_id[idx]\n",
    "  #print(parid)\n",
    "  # select row from original dataset to retrieve `text` and binary label\n",
    "  keyword = data.loc[data.par_id == parid].keyword.values[0]\n",
    "  text = data.loc[data.par_id == parid].text.values[0]\n",
    "  label = data.loc[data.par_id == parid].label.values[0]\n",
    "  rows.append({\n",
    "      'par_id':parid,\n",
    "      'community':keyword,\n",
    "      'text':text,\n",
    "      'label':label\n",
    "  })\n",
    "\n",
    "train_set = pd.DataFrame(rows)\n",
    "# Split train into train and internal validation set 80:20\n",
    "\n",
    "val_size = int(len(train_set) * 0.2)\n",
    "\n",
    "train_set = train_set.sample(frac=1)\n",
    "val_set = train_set.iloc[0:val_size].reset_index(drop=True).copy()\n",
    "train_set = train_set.iloc[val_size:].reset_index(drop=True).copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rebuild test set\n",
    "\n",
    "rows = [] # will contain par_id, label and text\n",
    "for idx in range(len(teids)):\n",
    "  parid = teids.par_id[idx]\n",
    "  #print(parid)\n",
    "  # select row from original dataset\n",
    "  keyword = data.loc[data.par_id == parid].keyword.values[0]\n",
    "  text = data.loc[data.par_id == parid].text.values[0]\n",
    "  label = data.loc[data.par_id == parid].label.values[0]\n",
    "  rows.append({\n",
    "      'par_id':parid,\n",
    "      'community':keyword,\n",
    "      'text':text,\n",
    "      'label':label\n",
    "  })\n",
    "\n",
    "test_set = pd.DataFrame(rows)\n",
    "test_set = test_set.sample(frac=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Internal validation set accuracy: 0.92\n",
      "Internal validation set F1 score: 0.3232323232323232\n"
     ]
    }
   ],
   "source": [
    "# Function to preprocess documents\n",
    "def preprocess(documents):\n",
    "    processed_docs = []\n",
    "    stop_words = set(stopwords.words(\"english\"))  # Load stop words once to improve efficiency\n",
    "    for document in documents:\n",
    "        words = word_tokenize(document.lower())\n",
    "        filtered_words = [word for word in words if word.isalpha() and word not in stop_words]\n",
    "        processed_docs.append(\" \".join(filtered_words))\n",
    "    return processed_docs\n",
    "\n",
    "# Preprocess documents for both train and dev sets\n",
    "documents_train = preprocess(train_set[\"text\"])\n",
    "documents_dev = preprocess(val_set[\"text\"])\n",
    "\n",
    "# Vectorize documents using CountVectorizer\n",
    "vectorizer = CountVectorizer()\n",
    "X_train = vectorizer.fit_transform(documents_train).toarray()\n",
    "y_train = train_set[\"label\"].values\n",
    "\n",
    "X_val = vectorizer.transform(documents_dev).toarray()\n",
    "y_val = val_set[\"label\"].values\n",
    "\n",
    "# Train logistic regression model\n",
    "clf = LogisticRegression(random_state=1, max_iter=1000).fit(X_train, y_train)\n",
    "\n",
    "# Evaluate model\n",
    "dev_score = clf.score(X_val, y_val)\n",
    "dev_predictions = clf.predict(X_val)\n",
    "dev_f1score = f1_score(y_val, dev_predictions)\n",
    "\n",
    "print(f\"Internal validation set accuracy: {dev_score}\")\n",
    "print(f\"Internal validation set F1 score: {dev_f1score}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "official dev set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test score accuracy: 0.9025322503583373\n",
      "Test F1 score: 0.25547445255474455\n"
     ]
    }
   ],
   "source": [
    "documents_test = preprocess(test_set[\"text\"])\n",
    "\n",
    "# Vectorize the test documents using the same vectorizer used for training\n",
    "X_test = vectorizer.transform(documents_test).toarray()\n",
    "y_test = test_set[\"label\"].values\n",
    "\n",
    "test_predictions = clf.predict(X_test)\n",
    "\n",
    "# Calculate evaluation metrics\n",
    "test_accuracy = clf.score(X_test, y_test)\n",
    "test_f1score = f1_score(y_test, test_predictions)\n",
    "\n",
    "print(f\"Test score accuracy: {test_accuracy}\")\n",
    "print(f\"Test F1 score: {test_f1score}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}