deBERTa_contextual_embeddings.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "Kxvk4OamxxVz"
      },
      "outputs": [],
      "source": [
        "# !pip install transformers[torch]\n",
        "# !pip install accelerate -U\n",
        "# !pip install nlpaug"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 563
        },
        "id": "nbkEulidY9Sx",
        "outputId": "d2b404aa-417f-476c-e032-f8cef4443aa4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n",
            "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       par_id      art_id     keyword country  \\\n",
              "0           1  @@24942188    hopeless      ph   \n",
              "1           2  @@21968160     migrant      gh   \n",
              "2           3  @@16584954   immigrant      ie   \n",
              "3           4   @@7811231    disabled      nz   \n",
              "4           5   @@1494111     refugee      ca   \n",
              "...       ...         ...         ...     ...   \n",
              "10464   10465  @@14297363       women      lk   \n",
              "10465   10466  @@70091353  vulnerable      ph   \n",
              "10466   10467  @@20282330     in-need      ng   \n",
              "10467   10468  @@16753236    hopeless      in   \n",
              "10468   10469  @@16779383    homeless      ie   \n",
              "\n",
              "                                                    text  label  \n",
              "0      We 're living in times of absolute insanity , ...      0  \n",
              "1      In Libya today , there are countless number of...      0  \n",
              "2      White House press secretary Sean Spicer said t...      0  \n",
              "3      Council customers only signs would be displaye...      0  \n",
              "4      \" Just like we received migrants fleeing El Sa...      0  \n",
              "...                                                  ...    ...  \n",
              "10464  Sri Lankan norms and culture inhibit women fro...      0  \n",
              "10465  He added that the AFP will continue to bank on...      0  \n",
              "10466  \" She has one huge platform , and information ...      1  \n",
              "10467  \" Anja Ringgren Loven I ca n't find a word to ...      1  \n",
              "10468  \" Guinness World Record of 540lbs of 7-layer m...      1  \n",
              "\n",
              "[10469 rows x 6 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4b82bc50-c836-43d8-8d1e-33c1ad0fd6b2\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>par_id</th>\n",
              "      <th>art_id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>country</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>@@24942188</td>\n",
              "      <td>hopeless</td>\n",
              "      <td>ph</td>\n",
              "      <td>We 're living in times of absolute insanity , ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>@@21968160</td>\n",
              "      <td>migrant</td>\n",
              "      <td>gh</td>\n",
              "      <td>In Libya today , there are countless number of...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>@@16584954</td>\n",
              "      <td>immigrant</td>\n",
              "      <td>ie</td>\n",
              "      <td>White House press secretary Sean Spicer said t...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>@@7811231</td>\n",
              "      <td>disabled</td>\n",
              "      <td>nz</td>\n",
              "      <td>Council customers only signs would be displaye...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>@@1494111</td>\n",
              "      <td>refugee</td>\n",
              "      <td>ca</td>\n",
              "      <td>\" Just like we received migrants fleeing El Sa...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10464</th>\n",
              "      <td>10465</td>\n",
              "      <td>@@14297363</td>\n",
              "      <td>women</td>\n",
              "      <td>lk</td>\n",
              "      <td>Sri Lankan norms and culture inhibit women fro...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10465</th>\n",
              "      <td>10466</td>\n",
              "      <td>@@70091353</td>\n",
              "      <td>vulnerable</td>\n",
              "      <td>ph</td>\n",
              "      <td>He added that the AFP will continue to bank on...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10466</th>\n",
              "      <td>10467</td>\n",
              "      <td>@@20282330</td>\n",
              "      <td>in-need</td>\n",
              "      <td>ng</td>\n",
              "      <td>\" She has one huge platform , and information ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10467</th>\n",
              "      <td>10468</td>\n",
              "      <td>@@16753236</td>\n",
              "      <td>hopeless</td>\n",
              "      <td>in</td>\n",
              "      <td>\" Anja Ringgren Loven I ca n't find a word to ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10468</th>\n",
              "      <td>10469</td>\n",
              "      <td>@@16779383</td>\n",
              "      <td>homeless</td>\n",
              "      <td>ie</td>\n",
              "      <td>\" Guinness World Record of 540lbs of 7-layer m...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10469 rows × 6 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4b82bc50-c836-43d8-8d1e-33c1ad0fd6b2')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-4b82bc50-c836-43d8-8d1e-33c1ad0fd6b2 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-4b82bc50-c836-43d8-8d1e-33c1ad0fd6b2');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-1ad7aa9f-7a90-46a5-9a53-ee3083faf142\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1ad7aa9f-7a90-46a5-9a53-ee3083faf142')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-1ad7aa9f-7a90-46a5-9a53-ee3083faf142 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "data",
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 10469,\n  \"fields\": [\n    {\n      \"column\": \"par_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3022,\n        \"min\": 1,\n        \"max\": 10469,\n        \"num_unique_values\": 10469,\n        \"samples\": [\n          7567,\n          7992,\n          2115\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"art_id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10093,\n        \"samples\": [\n          \"@@15858151\",\n          \"@@25187013\",\n          \"@@896186\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"keyword\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"women\",\n          \"migrant\",\n          \"in-need\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"country\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 20,\n        \"samples\": [\n          \"ph\",\n          \"lk\",\n          \"bd\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10468,\n        \"samples\": [\n          \"Ekso -- formerly known as Berkeley Bionics -- has been developing the Ekso exoskeleton for about seven years , beginning in 2005 with the Exohiker , a suit designed to assist load bearing . After discovering that energy efficiency meant a restriction of what they could do with their power , the company began looking at the disabled market to try and create a product that would help paraplegics walk again .\",\n          \"Thirty seven percent of the offences registered on Central Railway from January to May this year pertain to illegal travel in coaches reserved for the disabled .\",\n          \"Old buildings , coastal areas most vulnerable to major earthquake in B.C.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          1,\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import os\n",
        "import random\n",
        "\n",
        "from transformers import DebertaForSequenceClassification, DebertaTokenizer, Trainer, TrainingArguments\n",
        "import nltk\n",
        "import nlpaug.augmenter.word as naw\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.corpus import wordnet\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import f1_score, accuracy_score\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('omw-1.4')\n",
        "\n",
        "cuda_available = torch.cuda.is_available()\n",
        "device = 'cuda:0' if cuda_available else 'cpu'\n",
        "\n",
        "torch.manual_seed(0)\n",
        "\n",
        "data = pd.read_csv(\"dontpatronizeme_pcl.tsv\",\n",
        "                       sep=\"\\t\",\n",
        "                       names=['par_id', 'art_id', 'keyword', 'country', 'text', 'label'],\n",
        "                       skiprows=4)\n",
        "\n",
        "data['label'] = data['label'].apply(lambda x: 0 if x in [0, 1] else 1)\n",
        "\n",
        "trids = pd.read_csv('train_semeval_parids-labels.csv')\n",
        "teids = pd.read_csv('dev_semeval_parids-labels.csv')\n",
        "\n",
        "trids.par_id = trids.par_id\n",
        "teids.par_id = teids.par_id\n",
        "\n",
        "data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "MzzbZTz-xxV4"
      },
      "outputs": [],
      "source": [
        "# Rebuild training set\n",
        "\n",
        "rows = [] # will contain par_id, label and text\n",
        "for idx in range(len(trids)):\n",
        "  parid = trids.par_id[idx]\n",
        "  #print(parid)\n",
        "  # select row from original dataset to retrieve `text` and binary label\n",
        "  keyword = data.loc[data.par_id == parid].keyword.values[0]\n",
        "  text = data.loc[data.par_id == parid].text.values[0]\n",
        "  label = data.loc[data.par_id == parid].label.values[0]\n",
        "  rows.append({\n",
        "      'par_id':parid,\n",
        "      'community':keyword,\n",
        "      'text':text,\n",
        "      'label':label\n",
        "  })\n",
        "\n",
        "train_set = pd.DataFrame(rows)\n",
        "# Split train into train and internal validation set 80:20\n",
        "\n",
        "val_size = int(len(train_set) * 0.2)\n",
        "\n",
        "train_set = train_set.sample(frac=1)\n",
        "val_set = train_set.iloc[0:val_size].reset_index(drop=True).copy()\n",
        "train_set = train_set.iloc[val_size:].reset_index(drop=True).copy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "0Dz6kGVSaHws",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03afd1df-822a-40a5-f895-c36d431f2675"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "# Assuming 'train_set' is your training dataset and it's already prepared\n",
        "# Make sure to filter only positive examples as before\n",
        "train_data_positive = train_set[train_set[\"label\"] == 1]\n",
        "\n",
        "# Initialize the contextual word embeddings augmenter\n",
        "augmenter = naw.ContextualWordEmbsAug(\n",
        "    model_path='distilbert-base-uncased',  # Using DistilBERT for augmentation\n",
        "    action='substitute',  # We'll substitute words\n",
        "    top_k=20,  # Consider top 20 candidates for substitution\n",
        "    device=device\n",
        ")\n",
        "\n",
        "# Collect augmented data\n",
        "augmented_data_all = []\n",
        "\n",
        "# Loop through positive examples and augment\n",
        "for idx, row in train_data_positive.iterrows():\n",
        "    original_sentence = row[\"text\"]\n",
        "    # Augment each sentence 3 times\n",
        "    for _ in range(3):\n",
        "        augmented_sentence = augmenter.augment(original_sentence)[0]\n",
        "        # Create a new record for each augmented sentence\n",
        "        augmented_data = {\n",
        "            \"par_id\": row[\"par_id\"],  # Optionally, adjust if you want to track augmented data back to original\n",
        "            \"community\": row[\"community\"],  # Assuming 'keyword' is actually 'community' as per your dataset\n",
        "            \"text\": augmented_sentence,  # No need to select the first element, as each call returns a single augmentation\n",
        "            \"label\": 1  # Ensuring the label is correctly set for augmented data\n",
        "        }\n",
        "        augmented_data_all.append(augmented_data)\n",
        "\n",
        "# Convert augmented data into a DataFrame\n",
        "augmented_data_df = pd.DataFrame(augmented_data_all)\n",
        "\n",
        "# Concatenate original training set with augmented data\n",
        "train_set_augmented = pd.concat([train_set, augmented_data_df])\n",
        "\n",
        "# Shuffle the combined dataset\n",
        "train_set_augmented = train_set_augmented.sample(frac=1, random_state=1).reset_index(drop=True)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_set_augmented"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "YaO-vPvrJqCp",
        "outputId": "1f2cb42d-4c49-479f-f2b7-d90861978132"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      par_id   community                                               text  \\\n",
              "0       6781   immigrant  Peter Dutton is a post Fitzgerald ex-police of...   \n",
              "1       7506    homeless  it reveals the tragic fate of the arts movemen...   \n",
              "2       6686  vulnerable  Director of the RSU , Kenute Hare , cautioned ...   \n",
              "3       5820    hopeless  Coming from any other body this might seem hop...   \n",
              "4       9410  vulnerable  She was only 26 at the time and looks like a t...   \n",
              "...      ...         ...                                                ...   \n",
              "8567    3196       women  While Warriors ' spectators are expected to be...   \n",
              "8568    3721     in-need  children'em hospice, a program from child foun...   \n",
              "8569    4503     in-need  10 . Rather , it was friends of the deceased w...   \n",
              "8570    2932    hopeless  \" It dashes the hopes of the young people who ...   \n",
              "8571    3570     in-need  The rations will be distributed to those in ne...   \n",
              "\n",
              "      label  \n",
              "0         0  \n",
              "1         1  \n",
              "2         0  \n",
              "3         0  \n",
              "4         1  \n",
              "...     ...  \n",
              "8567      0  \n",
              "8568      1  \n",
              "8569      0  \n",
              "8570      0  \n",
              "8571      0  \n",
              "\n",
              "[8572 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e5958cd6-a443-4260-ac01-c28aab03c587\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>par_id</th>\n",
              "      <th>community</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>6781</td>\n",
              "      <td>immigrant</td>\n",
              "      <td>Peter Dutton is a post Fitzgerald ex-police of...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>7506</td>\n",
              "      <td>homeless</td>\n",
              "      <td>it reveals the tragic fate of the arts movemen...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>6686</td>\n",
              "      <td>vulnerable</td>\n",
              "      <td>Director of the RSU , Kenute Hare , cautioned ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>5820</td>\n",
              "      <td>hopeless</td>\n",
              "      <td>Coming from any other body this might seem hop...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>9410</td>\n",
              "      <td>vulnerable</td>\n",
              "      <td>She was only 26 at the time and looks like a t...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8567</th>\n",
              "      <td>3196</td>\n",
              "      <td>women</td>\n",
              "      <td>While Warriors ' spectators are expected to be...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8568</th>\n",
              "      <td>3721</td>\n",
              "      <td>in-need</td>\n",
              "      <td>children'em hospice, a program from child foun...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8569</th>\n",
              "      <td>4503</td>\n",
              "      <td>in-need</td>\n",
              "      <td>10 . Rather , it was friends of the deceased w...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8570</th>\n",
              "      <td>2932</td>\n",
              "      <td>hopeless</td>\n",
              "      <td>\" It dashes the hopes of the young people who ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8571</th>\n",
              "      <td>3570</td>\n",
              "      <td>in-need</td>\n",
              "      <td>The rations will be distributed to those in ne...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8572 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e5958cd6-a443-4260-ac01-c28aab03c587')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e5958cd6-a443-4260-ac01-c28aab03c587 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e5958cd6-a443-4260-ac01-c28aab03c587');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-38a05756-664f-42db-92bf-bc0813cb5cb2\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-38a05756-664f-42db-92bf-bc0813cb5cb2')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-38a05756-664f-42db-92bf-bc0813cb5cb2 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "train_set_augmented",
              "summary": "{\n  \"name\": \"train_set_augmented\",\n  \"rows\": 8572,\n  \"fields\": [\n    {\n      \"column\": \"par_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2669,\n        \"min\": 2,\n        \"max\": 10469,\n        \"num_unique_values\": 6700,\n        \"samples\": [\n          355,\n          3007,\n          1508\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"community\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"disabled\",\n          \"homeless\",\n          \"refugee\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 8572,\n        \"samples\": [\n          \"He said the residents , whose livestock were swept away by Pokot bandits , are now living hopeless lives in IDP camps without food , shelter and clothes .\",\n          \"Johannesburg - Strong winds , severe rain and flooding around Gauteng have left people on the East Rand homeless and have even brought Johannesburg 's art scene to a halt .\",\n          \"The president might be referring to a segment aired Friday night on the Fox News Channel show \\\" Tucker Carlson Tonight \\\" that reported Sweden had accepted more than 160,000 asylum-seekers last year but that only 500 of the migrants had found jobs in Sweden . The report went on to say that a surge in violence had followed .\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          1,\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "dekkuohSxxV5"
      },
      "outputs": [],
      "source": [
        "tokenizer = DebertaTokenizer.from_pretrained(\"microsoft/deberta-base\")\n",
        "train_encodings = tokenizer(list(train_set_augmented[\"text\"]), return_tensors=\"pt\", truncation=True, padding=True).to(device)\n",
        "val_encodings = tokenizer(list(val_set[\"text\"]), return_tensors=\"pt\", truncation=True, padding=True).to(device)\n",
        "train_labels = list(train_set_augmented[\"label\"])\n",
        "val_labels = list(val_set[\"label\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138,
          "referenced_widgets": [
            "166eb3580d2f42ffbe8d3e621c5a9bf4",
            "2c56ae8e4a8b4061a2af9d363b25ef56",
            "9df709ca10d249b9b778964c91d9efe0",
            "5b1351cf8eec4b7786533423195a7ce4",
            "93118757277d4fe3975198b613cf72ae",
            "2ab96444b0e94035b9e5bf00d38b6175",
            "70d6622ceb1e4072b940d004af8f4938",
            "78e83664ffe14e41bd934bcca9de4772",
            "ad5c3d0ca1244a608789e3db56c5aa5d",
            "20b86c42eaf74668bd3925c18dc76d2b",
            "2113d4daae5d4e1abf703d742a4dd1f3"
          ]
        },
        "id": "fhPqFVWLxxV6",
        "outputId": "cf70cd66-a5b6-4c9c-a3e0-79efa66c2e20"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pytorch_model.bin:   0%|          | 0.00/559M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "166eb3580d2f42ffbe8d3e621c5a9bf4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
            "  return self.fget.__get__(instance, owner)()\n",
            "Some weights of DebertaForSequenceClassification were not initialized from the model checkpoint at microsoft/deberta-base and are newly initialized: ['classifier.bias', 'classifier.weight', 'pooler.dense.bias', 'pooler.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "# Rebuild test set\n",
        "\n",
        "rows = [] # will contain par_id, label and text\n",
        "for idx in range(len(teids)):\n",
        "  parid = teids.par_id[idx]\n",
        "  #print(parid)\n",
        "  # select row from original dataset\n",
        "  keyword = data.loc[data.par_id == parid].keyword.values[0]\n",
        "  text = data.loc[data.par_id == parid].text.values[0]\n",
        "  label = data.loc[data.par_id == parid].label.values[0]\n",
        "  rows.append({\n",
        "      'par_id':parid,\n",
        "      'community':keyword,\n",
        "      'text':text,\n",
        "      'label':label\n",
        "  })\n",
        "\n",
        "test_set = pd.DataFrame(rows)\n",
        "test_set = test_set.sample(frac=1)\n",
        "\n",
        "class Dataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "train_dataset = Dataset(train_encodings, train_labels)\n",
        "val_dataset = Dataset(val_encodings, val_labels)\n",
        "\n",
        "# define model\n",
        "\n",
        "def compute_metrics(input):\n",
        "    y_pred = np.argmax(input.predictions, axis=1)\n",
        "    y_true = input.label_ids\n",
        "    accuracy = accuracy_score(y_true, y_pred)\n",
        "    f1score = f1_score(y_true, y_pred)\n",
        "    return {'accuracy': accuracy, 'f1 score': f1score}\n",
        "\n",
        "model = DebertaForSequenceClassification.from_pretrained(\"microsoft/deberta-base\", num_labels=2).to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 754
        },
        "id": "_eTHAvZSxxV7",
        "outputId": "890c5f8a-eb6d-4b19-e02d-c7cf99329a27"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-7-8d930fc6afba>:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='10720' max='10720' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [10720/10720 1:14:43, Epoch 10/10]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>F1 score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.226600</td>\n",
              "      <td>0.274198</td>\n",
              "      <td>0.908657</td>\n",
              "      <td>0.375510</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.172700</td>\n",
              "      <td>0.274941</td>\n",
              "      <td>0.914030</td>\n",
              "      <td>0.466667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.099600</td>\n",
              "      <td>0.433143</td>\n",
              "      <td>0.914030</td>\n",
              "      <td>0.414634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.034900</td>\n",
              "      <td>0.605049</td>\n",
              "      <td>0.912239</td>\n",
              "      <td>0.441065</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.020600</td>\n",
              "      <td>0.717788</td>\n",
              "      <td>0.896119</td>\n",
              "      <td>0.502857</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.004700</td>\n",
              "      <td>0.776876</td>\n",
              "      <td>0.907463</td>\n",
              "      <td>0.501608</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.005000</td>\n",
              "      <td>0.888838</td>\n",
              "      <td>0.907463</td>\n",
              "      <td>0.474576</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.001900</td>\n",
              "      <td>0.875992</td>\n",
              "      <td>0.909254</td>\n",
              "      <td>0.500000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.000800</td>\n",
              "      <td>0.923103</td>\n",
              "      <td>0.906866</td>\n",
              "      <td>0.486842</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.001600</td>\n",
              "      <td>0.937857</td>\n",
              "      <td>0.906269</td>\n",
              "      <td>0.478405</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-7-8d930fc6afba>:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-7-8d930fc6afba>:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-7-8d930fc6afba>:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-7-8d930fc6afba>:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-7-8d930fc6afba>:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-7-8d930fc6afba>:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-7-8d930fc6afba>:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-7-8d930fc6afba>:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
            "<ipython-input-7-8d930fc6afba>:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=10720, training_loss=0.06232314574160377, metrics={'train_runtime': 4484.6914, 'train_samples_per_second': 19.114, 'train_steps_per_second': 2.39, 'total_flos': 2.628154620616704e+16, 'train_loss': 0.06232314574160377, 'epoch': 10.0})"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "training_args = TrainingArguments(\n",
        "    output_dir=\"Deberta/\",\n",
        "    learning_rate=1e-5,\n",
        "    weight_decay=0.05,\n",
        "    per_device_train_batch_size=8,\n",
        "    per_device_eval_batch_size=8,\n",
        "    num_train_epochs=10,\n",
        "    do_eval=True,\n",
        "    evaluation_strategy=\"epoch\",\n",
        "    save_strategy=\"epoch\",\n",
        "    load_best_model_at_end=True,\n",
        "    metric_for_best_model=\"f1 score\",\n",
        "    greater_is_better=True,\n",
        "    report_to=\"none\"\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    tokenizer=tokenizer,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=val_dataset,\n",
        "    compute_metrics=compute_metrics\n",
        ")\n",
        "\n",
        "trainer.train()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "uXI-XOfbboRB"
      },
      "outputs": [],
      "source": [
        "trainer.save_model(f\"deberta_contextual_embedding\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "id": "pobvE6zRxxV8",
        "outputId": "fabf591d-2b97-4ee7-b301-e564419c0682"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-7-8d930fc6afba>:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": []
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Score: 0.5028571428571429\n"
          ]
        }
      ],
      "source": [
        "predictions = trainer.predict(val_dataset)\n",
        "\n",
        "# Step 1: Extract predictions and true labels\n",
        "raw_predictions = predictions.predictions\n",
        "true_labels = predictions.label_ids\n",
        "\n",
        "# Step 2: Convert logits to predicted labels\n",
        "predicted_labels = np.argmax(raw_predictions, axis=1)\n",
        "\n",
        "# Step 3: Compute the F1 score\n",
        "f1 = f1_score(true_labels, predicted_labels, average='binary')\n",
        "\n",
        "print(f\"F1 Score: {f1}\")\n",
        "# run predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_YIJ5zuesGgt",
        "outputId": "7ef6b394-3b73-46e7-878c-c4abb892a97f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "V100",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.8"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "166eb3580d2f42ffbe8d3e621c5a9bf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2c56ae8e4a8b4061a2af9d363b25ef56",
              "IPY_MODEL_9df709ca10d249b9b778964c91d9efe0",
              "IPY_MODEL_5b1351cf8eec4b7786533423195a7ce4"
            ],
            "layout": "IPY_MODEL_93118757277d4fe3975198b613cf72ae"
          }
        },
        "2c56ae8e4a8b4061a2af9d363b25ef56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2ab96444b0e94035b9e5bf00d38b6175",
            "placeholder": "​",
            "style": "IPY_MODEL_70d6622ceb1e4072b940d004af8f4938",
            "value": "pytorch_model.bin: 100%"
          }
        },
        "9df709ca10d249b9b778964c91d9efe0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78e83664ffe14e41bd934bcca9de4772",
            "max": 558614189,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ad5c3d0ca1244a608789e3db56c5aa5d",
            "value": 558614189
          }
        },
        "5b1351cf8eec4b7786533423195a7ce4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_20b86c42eaf74668bd3925c18dc76d2b",
            "placeholder": "​",
            "style": "IPY_MODEL_2113d4daae5d4e1abf703d742a4dd1f3",
            "value": " 559M/559M [00:05&lt;00:00, 97.3MB/s]"
          }
        },
        "93118757277d4fe3975198b613cf72ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2ab96444b0e94035b9e5bf00d38b6175": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "70d6622ceb1e4072b940d004af8f4938": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "78e83664ffe14e41bd934bcca9de4772": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ad5c3d0ca1244a608789e3db56c5aa5d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "20b86c42eaf74668bd3925c18dc76d2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2113d4daae5d4e1abf703d742a4dd1f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}