roBERTa_baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from simpletransformers.classification import ClassificationModel, ClassificationArgs, MultiLabelClassificationModel, MultiLabelClassificationArgs\n",
    "from urllib import request\n",
    "import pandas as pd\n",
    "import logging\n",
    "import torch\n",
    "import torchvision\n",
    "from collections import Counter\n",
    "from ast import literal_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cuda available? True\n"
     ]
    }
   ],
   "source": [
    "# prepare logger\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "\n",
    "transformers_logger = logging.getLogger(\"transformers\")\n",
    "transformers_logger.setLevel(logging.WARNING)\n",
    "\n",
    "# check gpu\n",
    "cuda_available = torch.cuda.is_available()\n",
    "\n",
    "print('Cuda available?', cuda_available)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper function to save predictions to an output file\n",
    "def labels2file(p, outf_path):\n",
    "\twith open(outf_path,'w') as outf:\n",
    "\t\tfor pi in p:\n",
    "\t\t\toutf.write(','.join([str(k) for k in pi])+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "def load_task():\n",
    "    \"\"\"\n",
    "    Load task 1 training set and convert the tags into binary labels. \n",
    "    Paragraphs with original labels of 0 or 1 are considered to be negative examples of PCL and will have the label 0 = negative.\n",
    "    Paragraphs with original labels of 2, 3 or 4 are considered to be positive examples of PCL and will have the label 1 = positive.\n",
    "    It returns a pandas dataframe with paragraphs and labels.\n",
    "    \"\"\"\n",
    "    rows=[]\n",
    "    with open(os.path.join('data/dontpatronizeme_pcl.tsv')) as f:\n",
    "        for line in f.readlines()[4:]:\n",
    "            par_id=line.strip().split('\\t')[0]\n",
    "            art_id = line.strip().split('\\t')[1]\n",
    "            keyword=line.strip().split('\\t')[2]\n",
    "            country=line.strip().split('\\t')[3]\n",
    "            t=line.strip().split('\\t')[4]#.lower()\n",
    "            l=line.strip().split('\\t')[-1]\n",
    "            if l=='0' or l=='1':\n",
    "                lbin=0\n",
    "            else:\n",
    "                lbin=1\n",
    "            rows.append(\n",
    "                {'par_id':par_id,\n",
    "                'art_id':art_id,\n",
    "                'keyword':keyword,\n",
    "                'country':country,\n",
    "                'text':t, \n",
    "                'label':lbin, \n",
    "                'orig_label':l\n",
    "                }\n",
    "                )\n",
    "    df=pd.DataFrame(rows, columns=['par_id', 'art_id', 'keyword', 'country', 'text', 'label', 'orig_label']) \n",
    "    return df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>par_id</th>\n",
       "      <th>art_id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>country</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>orig_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>@@24942188</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>ph</td>\n",
       "      <td>We 're living in times of absolute insanity , ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>@@21968160</td>\n",
       "      <td>migrant</td>\n",
       "      <td>gh</td>\n",
       "      <td>In Libya today , there are countless number of...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>@@16584954</td>\n",
       "      <td>immigrant</td>\n",
       "      <td>ie</td>\n",
       "      <td>\"White House press secretary Sean Spicer said ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>@@7811231</td>\n",
       "      <td>disabled</td>\n",
       "      <td>nz</td>\n",
       "      <td>Council customers only signs would be displaye...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>@@1494111</td>\n",
       "      <td>refugee</td>\n",
       "      <td>ca</td>\n",
       "      <td>\"\"\" Just like we received migrants fleeing El ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10464</th>\n",
       "      <td>10465</td>\n",
       "      <td>@@14297363</td>\n",
       "      <td>women</td>\n",
       "      <td>lk</td>\n",
       "      <td>\"Sri Lankan norms and culture inhibit women fr...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10465</th>\n",
       "      <td>10466</td>\n",
       "      <td>@@70091353</td>\n",
       "      <td>vulnerable</td>\n",
       "      <td>ph</td>\n",
       "      <td>He added that the AFP will continue to bank on...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10466</th>\n",
       "      <td>10467</td>\n",
       "      <td>@@20282330</td>\n",
       "      <td>in-need</td>\n",
       "      <td>ng</td>\n",
       "      <td>\"\"\" She has one huge platform , and informatio...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10467</th>\n",
       "      <td>10468</td>\n",
       "      <td>@@16753236</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>in</td>\n",
       "      <td>\"\"\" Anja Ringgren Loven I ca n't find a word t...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10468</th>\n",
       "      <td>10469</td>\n",
       "      <td>@@16779383</td>\n",
       "      <td>homeless</td>\n",
       "      <td>ie</td>\n",
       "      <td>\"\"\" Guinness World Record of 540lbs of 7-layer...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10469 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      par_id      art_id     keyword country  \\\n",
       "0          1  @@24942188    hopeless      ph   \n",
       "1          2  @@21968160     migrant      gh   \n",
       "2          3  @@16584954   immigrant      ie   \n",
       "3          4   @@7811231    disabled      nz   \n",
       "4          5   @@1494111     refugee      ca   \n",
       "...      ...         ...         ...     ...   \n",
       "10464  10465  @@14297363       women      lk   \n",
       "10465  10466  @@70091353  vulnerable      ph   \n",
       "10466  10467  @@20282330     in-need      ng   \n",
       "10467  10468  @@16753236    hopeless      in   \n",
       "10468  10469  @@16779383    homeless      ie   \n",
       "\n",
       "                                                    text  label orig_label  \n",
       "0      We 're living in times of absolute insanity , ...      0          0  \n",
       "1      In Libya today , there are countless number of...      0          0  \n",
       "2      \"White House press secretary Sean Spicer said ...      0          0  \n",
       "3      Council customers only signs would be displaye...      0          0  \n",
       "4      \"\"\" Just like we received migrants fleeing El ...      0          0  \n",
       "...                                                  ...    ...        ...  \n",
       "10464  \"Sri Lankan norms and culture inhibit women fr...      0          1  \n",
       "10465  He added that the AFP will continue to bank on...      0          0  \n",
       "10466  \"\"\" She has one huge platform , and informatio...      1          3  \n",
       "10467  \"\"\" Anja Ringgren Loven I ca n't find a word t...      1          4  \n",
       "10468  \"\"\" Guinness World Record of 540lbs of 7-layer...      1          3  \n",
       "\n",
       "[10469 rows x 7 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trids = pd.read_csv('data/train_semeval_parids-labels.csv')\n",
    "teids = pd.read_csv('data/dev_semeval_parids-labels.csv')\n",
    "\n",
    "trids.par_id = trids.par_id.astype(str)\n",
    "teids.par_id = teids.par_id.astype(str)\n",
    "\n",
    "data = load_task()\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>par_id</th>\n",
       "      <th>community</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4341</td>\n",
       "      <td>poor-families</td>\n",
       "      <td>The scheme saw an estimated 150,000 children f...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4136</td>\n",
       "      <td>homeless</td>\n",
       "      <td>Durban 's homeless communities reconciliation ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10352</td>\n",
       "      <td>poor-families</td>\n",
       "      <td>The next immediate problem that cropped up was...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8279</td>\n",
       "      <td>vulnerable</td>\n",
       "      <td>Far more important than the implications for t...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1164</td>\n",
       "      <td>poor-families</td>\n",
       "      <td>To strengthen child-sensitive social protectio...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8370</th>\n",
       "      <td>8380</td>\n",
       "      <td>refugee</td>\n",
       "      <td>Rescue teams search for survivors on the rubbl...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8371</th>\n",
       "      <td>8381</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>The launch of ' Happy Birthday ' took place la...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8372</th>\n",
       "      <td>8382</td>\n",
       "      <td>homeless</td>\n",
       "      <td>The unrest has left at least 20,000 people dea...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8373</th>\n",
       "      <td>8383</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>You have to see it from my perspective . I may...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8374</th>\n",
       "      <td>8384</td>\n",
       "      <td>disabled</td>\n",
       "      <td>Yet there was one occasion when we went to the...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8375 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     par_id      community                                               text  \\\n",
       "0      4341  poor-families  The scheme saw an estimated 150,000 children f...   \n",
       "1      4136       homeless  Durban 's homeless communities reconciliation ...   \n",
       "2     10352  poor-families  The next immediate problem that cropped up was...   \n",
       "3      8279     vulnerable  Far more important than the implications for t...   \n",
       "4      1164  poor-families  To strengthen child-sensitive social protectio...   \n",
       "...     ...            ...                                                ...   \n",
       "8370   8380        refugee  Rescue teams search for survivors on the rubbl...   \n",
       "8371   8381       hopeless  The launch of ' Happy Birthday ' took place la...   \n",
       "8372   8382       homeless  The unrest has left at least 20,000 people dea...   \n",
       "8373   8383       hopeless  You have to see it from my perspective . I may...   \n",
       "8374   8384       disabled  Yet there was one occasion when we went to the...   \n",
       "\n",
       "      label  \n",
       "0         1  \n",
       "1         1  \n",
       "2         1  \n",
       "3         1  \n",
       "4         1  \n",
       "...     ...  \n",
       "8370      0  \n",
       "8371      0  \n",
       "8372      0  \n",
       "8373      0  \n",
       "8374      0  \n",
       "\n",
       "[8375 rows x 4 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rebuild training set \n",
    "\n",
    "rows = [] # will contain par_id, label and text\n",
    "for idx in range(len(trids)):\n",
    "  parid = trids.par_id[idx]\n",
    "  #print(parid)\n",
    "  # select row from original dataset to retrieve `text` and binary label\n",
    "  keyword = data.loc[data.par_id == parid].keyword.values[0]\n",
    "  text = data.loc[data.par_id == parid].text.values[0]\n",
    "  label = data.loc[data.par_id == parid].label.values[0]\n",
    "  rows.append({\n",
    "      'par_id':parid,\n",
    "      'community':keyword,\n",
    "      'text':text,\n",
    "      'label':label\n",
    "  })\n",
    "\n",
    "trdf1 = pd.DataFrame(rows)\n",
    "trdf1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rebuild test set \n",
    "\n",
    "rows = [] # will contain par_id, label and text\n",
    "for idx in range(len(teids)):\n",
    "  parid = teids.par_id[idx]\n",
    "  #print(parid)\n",
    "  # select row from original dataset\n",
    "  keyword = data.loc[data.par_id == parid].keyword.values[0]\n",
    "  text = data.loc[data.par_id == parid].text.values[0]\n",
    "  label = data.loc[data.par_id == parid].label.values[0]\n",
    "  rows.append({\n",
    "      'par_id':parid,\n",
    "      'community':keyword,\n",
    "      'text':text,\n",
    "      'label':label\n",
    "  })\n",
    "  \n",
    "tedf1 = pd.DataFrame(rows)\n",
    "tedf1 = tedf1.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# downsample negative instances\n",
    "pcldf = trdf1[trdf1.label==1]\n",
    "npos = len(pcldf)\n",
    "\n",
    "training_set1 = pd.concat([pcldf, trdf1[trdf1.label==0][:npos*2]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "C:\\Users\\Tao\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\simpletransformers\\classification\\classification_model.py:610: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
      "  warnings.warn(\n",
      "INFO:simpletransformers.classification.classification_utils: Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c48295cdd6cc4b23acaa5d3ea37efac0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a026b7fe86449d99c30a67f0c63d129",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch:   0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f033a5fb8b64a09813a8b393fd09864",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 1 of 10:   0%|          | 0/298 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "230b462dcb944a8889574ea52a2561cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 2 of 10:   0%|          | 0/298 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8d70898c9c6476294c20db147be81a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 3 of 10:   0%|          | 0/298 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "333156a3ba8c4334ad777b173d048247",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 4 of 10:   0%|          | 0/298 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7f15e1276604f619b96e1b89990d486",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 5 of 10:   0%|          | 0/298 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06f765bf7e944e45a598ee9233f38d2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 6 of 10:   0%|          | 0/298 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fea710dabf2416ab536f667bd767f70",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 7 of 10:   0%|          | 0/298 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c07168182cb440aac70c4a1029d9618",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 8 of 10:   0%|          | 0/298 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32add49fefcf44cba08970f89a06c329",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 9 of 10:   0%|          | 0/298 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69f90c31c656493da7c4371e0ca5a7e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Epoch 10 of 10:   0%|          | 0/298 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:simpletransformers.classification.classification_model: Training of roberta model complete. Saved to outputs/.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2980, 0.21280165947833093)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "task1_model_args = ClassificationArgs(num_train_epochs=10,\n",
    "                                      no_save=True,\n",
    "                                      no_cache=True,\n",
    "                                      overwrite_output_dir=True)\n",
    "task1_model = ClassificationModel(\"roberta\",\n",
    "                                  'roberta-base',\n",
    "                                  args = task1_model_args,\n",
    "                                  num_labels=2,\n",
    "                                  use_cuda=cuda_available)\n",
    "# train model\n",
    "task1_model.train_model(training_set1[['text', 'label']])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:simpletransformers.classification.classification_utils: Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c0dc91bcf3843c6aebe7213b6569f72",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12562e212c8a48328fc9814ca7a0542e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/21 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RoBERTa baseline F1 Score: 0.5053003533568905\n"
     ]
    }
   ],
   "source": [
    "# run predictions\n",
    "preds_task1, _ = task1_model.predict(tedf1.text.tolist())\n",
    "\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "f1 = f1_score(tedf1.label.tolist(), preds_task1)\n",
    "\n",
    "print(f'RoBERTa baseline F1 Score: {f1}')\n",
    "\n",
    "# labels2file([[k] for k in preds], 'roberta_preds.txt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}