04-toxic_multilabel.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "toxic_multilabel.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "9fc7627a06b347b585e652916bbf6491": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_52a030b7dbb142cda029cbe017cb0822",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fba3c87d7dec4a4eb7ae75e739f3cf6e",
              "IPY_MODEL_138a1ed9015243c7a734ee6a5f7c9609"
            ]
          }
        },
        "52a030b7dbb142cda029cbe017cb0822": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fba3c87d7dec4a4eb7ae75e739f3cf6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_02d0482ff3d7468d9de60e8ce1ea1733",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3f61b3aaca65465c9b54c8513a5555e7"
          }
        },
        "138a1ed9015243c7a734ee6a5f7c9609": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_34e411387959477a957d6061f003e1f9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 302kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9532d9ec22784d519fc710e6189f89b8"
          }
        },
        "02d0482ff3d7468d9de60e8ce1ea1733": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3f61b3aaca65465c9b54c8513a5555e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "34e411387959477a957d6061f003e1f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9532d9ec22784d519fc710e6189f89b8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m5tUoHe9FRhs"
      },
      "source": [
        "## Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dr7BCHS-nIRW",
        "scrolled": true
      },
      "source": [
        "!pip install transformers\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import torch\n",
        "from torch.nn import BCEWithLogitsLoss, BCELoss\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import MultiLabelBinarizer\n",
        "from sklearn.metrics import classification_report, confusion_matrix, multilabel_confusion_matrix, f1_score, accuracy_score\n",
        "import pickle\n",
        "from transformers import *\n",
        "from tqdm import tqdm, trange\n",
        "from ast import literal_eval"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zB5Dij6JuItX"
      },
      "source": [
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UhjnJEwKnISB",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e342121d-885a-48f1-cd4a-5f8a01f579dc"
      },
      "source": [
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  raise SystemError('GPU device not found')\n",
        "print('Found GPU at: {}'.format(device_name))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uorMX_zrnISM",
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "48a34b4e-183c-465f-a096-da833989b59e"
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "n_gpu = torch.cuda.device_count()\n",
        "torch.cuda.get_device_name(0)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Tesla P100-PCIE-16GB'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dLcetMjZFjSH"
      },
      "source": [
        "## Load and Preprocess Training Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2dal0ggBcYdD"
      },
      "source": [
        "Dataset will be tokenized then split into training and validation sets. The validation set will be used to monitor training. For testing a separate test set will be loaded for analysis."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0ecREc7GnISW",
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "54a37dd4-18da-4120-94f6-9e3f32d0f5ce"
      },
      "source": [
        "df = pd.read_csv('train.csv') #jigsaw-toxic-comment-classification-challenge\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0000997932d777bf</td>\n",
              "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000103f0d9cfb60f</td>\n",
              "      <td>D'aww! He matches this background colour I'm s...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>000113f07ec002fd</td>\n",
              "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0001b41b1c6bb37e</td>\n",
              "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0001d958c54c6e35</td>\n",
              "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 id  ... identity_hate\n",
              "0  0000997932d777bf  ...             0\n",
              "1  000103f0d9cfb60f  ...             0\n",
              "2  000113f07ec002fd  ...             0\n",
              "3  0001b41b1c6bb37e  ...             0\n",
              "4  0001d958c54c6e35  ...             0\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6AhWrzX7nITB",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "cd5acbc1-525c-419f-df32-4514130dd3ef"
      },
      "source": [
        "print('Unique comments: ', df.comment_text.nunique() == df.shape[0])\n",
        "print('Null values: ', df.isnull().values.any())\n",
        "# df[df.isna().any(axis=1)]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Unique comments:  True\n",
            "Null values:  False\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OkKpz_9eJRt7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "c473c234-1472-42b3-e9e7-af36a96503f3"
      },
      "source": [
        "print('average sentence length: ', df.comment_text.str.split().str.len().mean())\n",
        "print('stdev sentence length: ', df.comment_text.str.split().str.len().std())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "average sentence length:  67.27352714465661\n",
            "stdev sentence length:  99.2307021928862\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UVI59S9VaAfB",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0a3d5338-0cb7-40d1-ffe1-124d436afd9d"
      },
      "source": [
        "cols = df.columns\n",
        "label_cols = list(cols[2:])\n",
        "num_labels = len(label_cols)\n",
        "print('Label columns: ', label_cols)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Label columns:  ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xzgA5qQgYIBZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        },
        "outputId": "6a2937a6-445e-421c-dd53-34569e654f63"
      },
      "source": [
        "print('Count of 1 per label: \\n', df[label_cols].sum(), '\\n') # Label counts, may need to downsample or upsample\n",
        "print('Count of 0 per label: \\n', df[label_cols].eq(0).sum())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Count of 1 per label: \n",
            " toxic            15294\n",
            "severe_toxic      1595\n",
            "obscene           8449\n",
            "threat             478\n",
            "insult            7877\n",
            "identity_hate     1405\n",
            "dtype: int64 \n",
            "\n",
            "Count of 0 per label: \n",
            " toxic            144277\n",
            "severe_toxic     157976\n",
            "obscene          151122\n",
            "threat           159093\n",
            "insult           151694\n",
            "identity_hate    158166\n",
            "dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uFpSd4JzaAae"
      },
      "source": [
        "df = df.sample(frac=1).reset_index(drop=True) #shuffle rows"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0DF3ddjej5vd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "b22c40b3-bb87-4911-b21c-d4e4e6beeee0"
      },
      "source": [
        "df['one_hot_labels'] = list(df[label_cols].values)\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "      <th>one_hot_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>5a7ca836c0f0ad72</td>\n",
              "      <td>\"\\nIt should be obvious that that it is not th...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ebdf69f2b2f08828</td>\n",
              "      <td>I see you back from Madison</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>a492014f9b9f739f</td>\n",
              "      <td>The topic appears to meet the criteria for spe...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>e1e455df19ad4130</td>\n",
              "      <td>14Feb07\\n\\nCristina,\\n\\nAm I allowed to wish y...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>b86bf32110384b88</td>\n",
              "      <td>Now all the margins are trashed from restoring...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 id  ...      one_hot_labels\n",
              "0  5a7ca836c0f0ad72  ...  [0, 0, 0, 0, 0, 0]\n",
              "1  ebdf69f2b2f08828  ...  [0, 0, 0, 0, 0, 0]\n",
              "2  a492014f9b9f739f  ...  [0, 0, 0, 0, 0, 0]\n",
              "3  e1e455df19ad4130  ...  [0, 0, 0, 0, 0, 0]\n",
              "4  b86bf32110384b88  ...  [0, 0, 0, 0, 0, 0]\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MlhHifh5bW7e"
      },
      "source": [
        "labels = list(df.one_hot_labels.values)\n",
        "comments = list(df.comment_text.values)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IlMHfElhGJzc"
      },
      "source": [
        "Load the pretrained tokenizer that corresponds to your choice in model. e.g.,\n",
        "\n",
        "```\n",
        "BERT:\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True) \n",
        "\n",
        "XLNet:\n",
        "tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased', do_lower_case=False) \n",
        "\n",
        "RoBERTa:\n",
        "tokenizer = RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=False)\n",
        "```\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IhVr8SziL_PY"
      },
      "source": [
        "In order to avoid memory issues with Google Colab, I enforce a max_length of 100 tokens. Note that some sentences may not adequately represent each label because of this."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HNsEu-vUur-4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83,
          "referenced_widgets": [
            "9fc7627a06b347b585e652916bbf6491",
            "52a030b7dbb142cda029cbe017cb0822",
            "fba3c87d7dec4a4eb7ae75e739f3cf6e",
            "138a1ed9015243c7a734ee6a5f7c9609",
            "02d0482ff3d7468d9de60e8ce1ea1733",
            "3f61b3aaca65465c9b54c8513a5555e7",
            "34e411387959477a957d6061f003e1f9",
            "9532d9ec22784d519fc710e6189f89b8"
          ]
        },
        "outputId": "273a8056-4b12-4c91-8941-c53377b629b5"
      },
      "source": [
        "max_length = 100\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True) # tokenizer\n",
        "encodings = tokenizer.batch_encode_plus(comments,max_length=max_length,pad_to_max_length=True) # tokenizer's encoding method\n",
        "print('tokenizer outputs: ', encodings.keys())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9fc7627a06b347b585e652916bbf6491",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "tokenizer outputs:  dict_keys(['input_ids', 'token_type_ids', 'attention_mask'])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l6CCLSjfur-9"
      },
      "source": [
        "input_ids = encodings['input_ids'] # tokenized and encoded sentences\n",
        "token_type_ids = encodings['token_type_ids'] # token type ids\n",
        "attention_masks = encodings['attention_mask'] # attention masks"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vSOFbThlYcpb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ba3e9ca0-31b4-4e0d-dd34-a6a28c2fc4c0"
      },
      "source": [
        "# Identifying indices of 'one_hot_labels' entries that only occur once - this will allow us to stratify split our training data later\n",
        "label_counts = df.one_hot_labels.astype(str).value_counts()\n",
        "one_freq = label_counts[label_counts==1].keys()\n",
        "one_freq_idxs = sorted(list(df[df.one_hot_labels.astype(str).isin(one_freq)].index), reverse=True)\n",
        "print('df label indices with only one instance: ', one_freq_idxs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "df label indices with only one instance:  [86236, 73729]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQQ7CoOag_r7"
      },
      "source": [
        "# Gathering single instance inputs to force into the training set after stratified split\n",
        "one_freq_input_ids = [input_ids.pop(i) for i in one_freq_idxs]\n",
        "one_freq_token_types = [token_type_ids.pop(i) for i in one_freq_idxs]\n",
        "one_freq_attention_masks = [attention_masks.pop(i) for i in one_freq_idxs]\n",
        "one_freq_labels = [labels.pop(i) for i in one_freq_idxs]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r9PxAt48HRRj"
      },
      "source": [
        "Be sure to handle all classes during validation using \"stratify\" during train/validation split:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WPFaq4ufnIT2"
      },
      "source": [
        "# Use train_test_split to split our data into train and validation sets\n",
        "\n",
        "train_inputs, validation_inputs, train_labels, validation_labels, train_token_types, validation_token_types, train_masks, validation_masks = train_test_split(input_ids, labels, token_type_ids,attention_masks,\n",
        "                                                            random_state=2020, test_size=0.10, stratify = labels)\n",
        "\n",
        "# Add one frequency data to train data\n",
        "train_inputs.extend(one_freq_input_ids)\n",
        "train_labels.extend(one_freq_labels)\n",
        "train_masks.extend(one_freq_attention_masks)\n",
        "train_token_types.extend(one_freq_token_types)\n",
        "\n",
        "# Convert all of our data into torch tensors, the required datatype for our model\n",
        "train_inputs = torch.tensor(train_inputs)\n",
        "train_labels = torch.tensor(train_labels)\n",
        "train_masks = torch.tensor(train_masks)\n",
        "train_token_types = torch.tensor(train_token_types)\n",
        "\n",
        "validation_inputs = torch.tensor(validation_inputs)\n",
        "validation_labels = torch.tensor(validation_labels)\n",
        "validation_masks = torch.tensor(validation_masks)\n",
        "validation_token_types = torch.tensor(validation_token_types)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZRnuLna-nIT4"
      },
      "source": [
        "# Select a batch size for training. For fine-tuning with XLNet, the authors recommend a batch size of 32, 48, or 128. We will use 32 here to avoid memory issues.\n",
        "batch_size = 32\n",
        "\n",
        "# Create an iterator of our data with torch DataLoader. This helps save on memory during training because, unlike a for loop, \n",
        "# with an iterator the entire dataset does not need to be loaded into memory\n",
        "\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels, train_token_types)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels, validation_token_types)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iiFRnP_ZTBFa"
      },
      "source": [
        "torch.save(validation_dataloader,'validation_data_loader')\n",
        "torch.save(train_dataloader,'train_data_loader')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ncGteBuSFuZM"
      },
      "source": [
        "## Load Model & Set Params"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z0dL-Bz_NrGj"
      },
      "source": [
        "Load the appropriate model below, each model already contains a single dense layer for classification on top.\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "BERT:\n",
        "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=num_labels)\n",
        "\n",
        "XLNet:\n",
        "model = XLNetForSequenceClassification.from_pretrained(\"xlnet-base-cased\", num_labels=num_labels)\n",
        "\n",
        "RoBERTa:\n",
        "model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=num_labels)\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ujk4k16DnIT6"
      },
      "source": [
        "# Load model, the pretrained model will include a single linear classification layer on top for classification. \n",
        "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=num_labels)\n",
        "model.cuda()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jGE4gv9qfhRG"
      },
      "source": [
        "Setting custom optimization parameters for the AdamW optimizer https://huggingface.co/transformers/main_classes/optimizer_schedules.html"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GsV8zwWYnIT9"
      },
      "source": [
        "# setting custom optimization parameters. You may implement a scheduler here as well.\n",
        "param_optimizer = list(model.named_parameters())\n",
        "no_decay = ['bias', 'gamma', 'beta']\n",
        "optimizer_grouped_parameters = [\n",
        "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.01},\n",
        "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.0}\n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aOomZIEIoHOL"
      },
      "source": [
        "optimizer = AdamW(optimizer_grouped_parameters,lr=2e-5,correct_bias=True)\n",
        "# optimizer = AdamW(model.parameters(),lr=2e-5)  # Default optimization"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JRQQZ8zIFzLW"
      },
      "source": [
        "## Train Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uDLZmEC_oKo3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "fba51964-6ff1-44f0-d448-d80c3824b6d9"
      },
      "source": [
        "# Store our loss and accuracy for plotting\n",
        "train_loss_set = []\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 3\n",
        "\n",
        "# trange is a tqdm wrapper around the normal python range\n",
        "for _ in trange(epochs, desc=\"Epoch\"):\n",
        "\n",
        "  # Training\n",
        "  \n",
        "  # Set our model to training mode (as opposed to evaluation mode)\n",
        "  model.train()\n",
        "\n",
        "  # Tracking variables\n",
        "  tr_loss = 0 #running loss\n",
        "  nb_tr_examples, nb_tr_steps = 0, 0\n",
        "  \n",
        "  # Train the data for one epoch\n",
        "  for step, batch in enumerate(train_dataloader):\n",
        "    # Add batch to GPU\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels, b_token_types = batch\n",
        "    # Clear out the gradients (by default they accumulate)\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # # Forward pass for multiclass classification\n",
        "    # outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "    # loss = outputs[0]\n",
        "    # logits = outputs[1]\n",
        "\n",
        "    # Forward pass for multilabel classification\n",
        "    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    logits = outputs[0]\n",
        "    loss_func = BCEWithLogitsLoss() \n",
        "    loss = loss_func(logits.view(-1,num_labels),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    # loss_func = BCELoss() \n",
        "    # loss = loss_func(torch.sigmoid(logits.view(-1,num_labels)),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    train_loss_set.append(loss.item())    \n",
        "\n",
        "    # Backward pass\n",
        "    loss.backward()\n",
        "    # Update parameters and take a step using the computed gradient\n",
        "    optimizer.step()\n",
        "    # scheduler.step()\n",
        "    # Update tracking variables\n",
        "    tr_loss += loss.item()\n",
        "    nb_tr_examples += b_input_ids.size(0)\n",
        "    nb_tr_steps += 1\n",
        "\n",
        "  print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
        "\n",
        "###############################################################################\n",
        "\n",
        "  # Validation\n",
        "\n",
        "  # Put model in evaluation mode to evaluate loss on the validation set\n",
        "  model.eval()\n",
        "\n",
        "  # Variables to gather full output\n",
        "  logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "  # Predict\n",
        "  for i, batch in enumerate(validation_dataloader):\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels, b_token_types = batch\n",
        "    with torch.no_grad():\n",
        "      # Forward pass\n",
        "      outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "      b_logit_pred = outs[0]\n",
        "      pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "      b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "      pred_label = pred_label.to('cpu').numpy()\n",
        "      b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "    tokenized_texts.append(b_input_ids)\n",
        "    logit_preds.append(b_logit_pred)\n",
        "    true_labels.append(b_labels)\n",
        "    pred_labels.append(pred_label)\n",
        "\n",
        "  # Flatten outputs\n",
        "  pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "  true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "  # Calculate Accuracy\n",
        "  threshold = 0.50\n",
        "  pred_bools = [pl>threshold for pl in pred_labels]\n",
        "  true_bools = [tl==1 for tl in true_labels]\n",
        "  val_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')*100\n",
        "  val_flat_accuracy = accuracy_score(true_bools, pred_bools)*100\n",
        "\n",
        "  print('F1 Validation Accuracy: ', val_f1_accuracy)\n",
        "  print('Flat Validation Accuracy: ', val_flat_accuracy)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\rEpoch:   0%|          | 0/3 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train loss: 0.05082032523087247\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rEpoch:  33%|███▎      | 1/3 [23:00<46:01, 1380.97s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "F1 Validation Accuracy:  78.08238436665262\n",
            "Flat Validation Accuracy:  92.33565206492449\n",
            "Train loss: 0.03456845487250288\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rEpoch:  67%|██████▋   | 2/3 [46:01<23:00, 1380.97s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "F1 Validation Accuracy:  78.78084179970972\n",
            "Flat Validation Accuracy:  92.59885943473084\n",
            "Train loss: 0.027323115857782174\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 100%|██████████| 3/3 [1:09:02<00:00, 1380.97s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "F1 Validation Accuracy:  78.47726262775885\n",
            "Flat Validation Accuracy:  92.8495331202607\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aiBeiBSRoOuz"
      },
      "source": [
        "torch.save(model.state_dict(), 'bert_model_toxic')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_7dd2GE3F4yK"
      },
      "source": [
        "## Load and Preprocess Test Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G5Q7hC4GFOLJ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "outputId": "f74c59f3-1030-4ecd-9988-bdbf079b9776"
      },
      "source": [
        "test_df = pd.read_csv('test.csv')\n",
        "test_labels_df = pd.read_csv('test_labels.csv')\n",
        "test_df = test_df.merge(test_labels_df, on='id', how='left')\n",
        "test_label_cols = list(test_df.columns[2:])\n",
        "print('Null values: ', test_df.isnull().values.any()) #should not be any null sentences or labels\n",
        "print('Same columns between train and test: ', label_cols == test_label_cols) #columns should be the same\n",
        "test_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Null values:  False\n",
            "Same columns between train and test:  True\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>00001cee341fdb12</td>\n",
              "      <td>Yo bitch Ja Rule is more succesful then you'll...</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0000247867823ef7</td>\n",
              "      <td>== From RfC == \\n\\n The title is fine as it is...</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>00013b17ad220c46</td>\n",
              "      <td>\" \\n\\n == Sources == \\n\\n * Zawe Ashton on Lap...</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>00017563c3f7919a</td>\n",
              "      <td>:If you have a look back at the source, the in...</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>00017695ad8997eb</td>\n",
              "      <td>I don't anonymously edit articles at all.</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 id  ... identity_hate\n",
              "0  00001cee341fdb12  ...            -1\n",
              "1  0000247867823ef7  ...            -1\n",
              "2  00013b17ad220c46  ...            -1\n",
              "3  00017563c3f7919a  ...            -1\n",
              "4  00017695ad8997eb  ...            -1\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "77rjCrMGpYxz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "c5f53ba7-7433-403f-906e-009dc8efc29f"
      },
      "source": [
        "test_df = test_df[~test_df[test_label_cols].eq(-1).any(axis=1)] #remove irrelevant rows/comments with -1 values\n",
        "test_df['one_hot_labels'] = list(test_df[test_label_cols].values)\n",
        "test_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "      <th>one_hot_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0001ea8717f6de06</td>\n",
              "      <td>Thank you for understanding. I think very high...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>000247e83dcc1211</td>\n",
              "      <td>:Dear god this site is horrible.</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>0002f87b16116a7f</td>\n",
              "      <td>\"::: Somebody will invariably try to add Relig...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>0003e1cccfd5a40a</td>\n",
              "      <td>\" \\n\\n It says it right there that it IS a typ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>00059ace3e3e9a53</td>\n",
              "      <td>\" \\n\\n == Before adding a new product to the l...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                  id  ...      one_hot_labels\n",
              "5   0001ea8717f6de06  ...  [0, 0, 0, 0, 0, 0]\n",
              "7   000247e83dcc1211  ...  [0, 0, 0, 0, 0, 0]\n",
              "11  0002f87b16116a7f  ...  [0, 0, 0, 0, 0, 0]\n",
              "13  0003e1cccfd5a40a  ...  [0, 0, 0, 0, 0, 0]\n",
              "14  00059ace3e3e9a53  ...  [0, 0, 0, 0, 0, 0]\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1a41OmU2i7qp"
      },
      "source": [
        "# Gathering input data\n",
        "test_labels = list(test_df.one_hot_labels.values)\n",
        "test_comments = list(test_df.comment_text.values)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "amySMO8EQzf2"
      },
      "source": [
        "# Encoding input data\n",
        "test_encodings = tokenizer.batch_encode_plus(test_comments,max_length=max_length,pad_to_max_length=True)\n",
        "test_input_ids = test_encodings['input_ids']\n",
        "test_token_type_ids = test_encodings['token_type_ids']\n",
        "test_attention_masks = test_encodings['attention_mask']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hqOfi9fkRaRN"
      },
      "source": [
        "# Make tensors out of data\n",
        "test_inputs = torch.tensor(test_input_ids)\n",
        "test_labels = torch.tensor(test_labels)\n",
        "test_masks = torch.tensor(test_attention_masks)\n",
        "test_token_types = torch.tensor(test_token_type_ids)\n",
        "# Create test dataloader\n",
        "test_data = TensorDataset(test_inputs, test_masks, test_labels, test_token_types)\n",
        "test_sampler = SequentialSampler(test_data)\n",
        "test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size)\n",
        "# Save test dataloader\n",
        "torch.save(test_dataloader,'test_data_loader')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PFTWxCA_GBau"
      },
      "source": [
        "## Prediction and Metics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NPvrL6OFSQvf"
      },
      "source": [
        "# Test\n",
        "\n",
        "# Put model in evaluation mode to evaluate loss on the validation set\n",
        "model.eval()\n",
        "\n",
        "#track variables\n",
        "logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "# Predict\n",
        "for i, batch in enumerate(test_dataloader):\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  # Unpack the inputs from our dataloader\n",
        "  b_input_ids, b_input_mask, b_labels, b_token_types = batch\n",
        "  with torch.no_grad():\n",
        "    # Forward pass\n",
        "    outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    b_logit_pred = outs[0]\n",
        "    pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "    b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "    pred_label = pred_label.to('cpu').numpy()\n",
        "    b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "  tokenized_texts.append(b_input_ids)\n",
        "  logit_preds.append(b_logit_pred)\n",
        "  true_labels.append(b_labels)\n",
        "  pred_labels.append(pred_label)\n",
        "\n",
        "# Flatten outputs\n",
        "tokenized_texts = [item for sublist in tokenized_texts for item in sublist]\n",
        "pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "true_labels = [item for sublist in true_labels for item in sublist]\n",
        "# Converting flattened binary values to boolean values\n",
        "true_bools = [tl==1 for tl in true_labels]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bQeGWqeMzAoZ"
      },
      "source": [
        "We need to threshold our sigmoid function outputs which range from [0, 1]. Below I use 0.50 as a threshold."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BZcZUcYOxxmM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 394
        },
        "outputId": "abc926d2-3b6d-432c-83af-76ebb8652e90"
      },
      "source": [
        "pred_bools = [pl>0.50 for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test F1 Accuracy:  0.6792739193783389\n",
            "Test Flat Accuracy:  0.8803651255118947 \n",
            "\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "        toxic       0.57      0.86      0.69      6090\n",
            " severe_toxic       0.40      0.47      0.43       367\n",
            "      obscene       0.64      0.78      0.70      3691\n",
            "       threat       0.45      0.70      0.55       211\n",
            "       insult       0.69      0.70      0.69      3427\n",
            "identity_hate       0.75      0.49      0.59       712\n",
            "\n",
            "    micro avg       0.61      0.77      0.68     14498\n",
            "    macro avg       0.58      0.67      0.61     14498\n",
            " weighted avg       0.62      0.77      0.68     14498\n",
            "  samples avg       0.07      0.07      0.07     14498\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5rLqrHK87eir"
      },
      "source": [
        "## Output Dataframe"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CJBkRdGN1hzx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b0301daf-0966-4060-c68c-2181cc43b3c3"
      },
      "source": [
        "idx2label = dict(zip(range(6),label_cols))\n",
        "print(idx2label)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{0: 'toxic', 1: 'severe_toxic', 2: 'obscene', 3: 'threat', 4: 'insult', 5: 'identity_hate'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QZUglV_A4BF_"
      },
      "source": [
        "# Getting indices of where boolean one hot vector true_bools is True so we can use idx2label to gather label names\n",
        "true_label_idxs, pred_label_idxs=[],[]\n",
        "for vals in true_bools:\n",
        "  true_label_idxs.append(np.where(vals)[0].flatten().tolist())\n",
        "for vals in pred_bools:\n",
        "  pred_label_idxs.append(np.where(vals)[0].flatten().tolist())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OOGhXM3R4a91"
      },
      "source": [
        "# Gathering vectors of label names using idx2label\n",
        "true_label_texts, pred_label_texts = [], []\n",
        "for vals in true_label_idxs:\n",
        "  if vals:\n",
        "    true_label_texts.append([idx2label[val] for val in vals])\n",
        "  else:\n",
        "    true_label_texts.append(vals)\n",
        "\n",
        "for vals in pred_label_idxs:\n",
        "  if vals:\n",
        "    pred_label_texts.append([idx2label[val] for val in vals])\n",
        "  else:\n",
        "    pred_label_texts.append(vals)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5HaqV6pn_HCG"
      },
      "source": [
        "# Decoding input ids to comment text\n",
        "comment_texts = [tokenizer.decode(text,skip_special_tokens=True,clean_up_tokenization_spaces=False) for text in tokenized_texts]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R7kk0Mgl1L-T",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "651efb03-73d7-4627-e1bf-a74c0e90066e"
      },
      "source": [
        "# Converting lists to df\n",
        "comparisons_df = pd.DataFrame({'comment_text': comment_texts, 'true_labels': true_label_texts, 'pred_labels':pred_label_texts})\n",
        "comparisons_df.to_csv('comparisons.csv')\n",
        "comparisons_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>comment_text</th>\n",
              "      <th>true_labels</th>\n",
              "      <th>pred_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>thank you for understanding . i think very hig...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>: dear god this site is horrible .</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>\" : : : somebody will invariably try to add re...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>\" it says it right there that it is a type . t...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>\" = = before adding a new product to the list ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                        comment_text true_labels pred_labels\n",
              "0  thank you for understanding . i think very hig...          []          []\n",
              "1                 : dear god this site is horrible .          []          []\n",
              "2  \" : : : somebody will invariably try to add re...          []          []\n",
              "3  \" it says it right there that it is a type . t...          []          []\n",
              "4  \" = = before adding a new product to the list ...          []          []"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PWFd18u3zlF8"
      },
      "source": [
        "## Bonus - Optimizing threshold value for micro F1 score"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c6mDy1lw0S4y"
      },
      "source": [
        "Doing this may result in a trade offs between precision, flat accuracy and micro F1 accuracy. You may tune the threshold however you want."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PHlhb2lvar8V",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "outputId": "f5773e91-f6b5-47bd-b235-63e1140a178a"
      },
      "source": [
        "# Calculate Accuracy - maximize F1 accuracy by tuning threshold values. First with 'macro_thresholds' on the order of e^-1 then with 'micro_thresholds' on the order of e^-2\n",
        "\n",
        "macro_thresholds = np.array(range(1,10))/10\n",
        "\n",
        "f1_results, flat_acc_results = [], []\n",
        "for th in macro_thresholds:\n",
        "  pred_bools = [pl>th for pl in pred_labels]\n",
        "  test_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')\n",
        "  test_flat_accuracy = accuracy_score(true_bools, pred_bools)\n",
        "  f1_results.append(test_f1_accuracy)\n",
        "  flat_acc_results.append(test_flat_accuracy)\n",
        "\n",
        "best_macro_th = macro_thresholds[np.argmax(f1_results)] #best macro threshold value\n",
        "\n",
        "micro_thresholds = (np.array(range(10))/100)+best_macro_th #calculating micro threshold values\n",
        "\n",
        "f1_results, flat_acc_results = [], []\n",
        "for th in micro_thresholds:\n",
        "  pred_bools = [pl>th for pl in pred_labels]\n",
        "  test_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')\n",
        "  test_flat_accuracy = accuracy_score(true_bools, pred_bools)\n",
        "  f1_results.append(test_f1_accuracy)\n",
        "  flat_acc_results.append(test_flat_accuracy)\n",
        "\n",
        "best_f1_idx = np.argmax(f1_results) #best threshold value\n",
        "\n",
        "# Printing and saving classification report\n",
        "print('Best Threshold: ', micro_thresholds[best_f1_idx])\n",
        "print('Test F1 Accuracy: ', f1_results[best_f1_idx])\n",
        "print('Test Flat Accuracy: ', flat_acc_results[best_f1_idx], '\\n')\n",
        "\n",
        "best_pred_bools = [pl>micro_thresholds[best_f1_idx] for pl in pred_labels]\n",
        "clf_report_optimized = classification_report(true_bools,best_pred_bools, target_names=label_cols)\n",
        "pickle.dump(clf_report_optimized, open('classification_report_optimized.txt','wb'))\n",
        "print(clf_report_optimized)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Best Threshold:  0.6\n",
            "Test F1 Accuracy:  0.6838089672413233\n",
            "Test Flat Accuracy:  0.8889149395104567 \n",
            "\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "        toxic       0.60      0.83      0.70      6090\n",
            " severe_toxic       0.47      0.34      0.39       367\n",
            "      obscene       0.68      0.74      0.71      3691\n",
            "       threat       0.52      0.63      0.57       211\n",
            "       insult       0.73      0.63      0.68      3427\n",
            "identity_hate       0.79      0.41      0.54       712\n",
            "\n",
            "    micro avg       0.65      0.73      0.68     14498\n",
            "    macro avg       0.63      0.60      0.60     14498\n",
            " weighted avg       0.66      0.73      0.68     14498\n",
            "  samples avg       0.07      0.07      0.07     14498\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oBg6UCYAYtIe"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}