Pooled RCNN performance analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we will test a pooled RCNN model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dXsRFoKJSNrH"
   },
   "source": [
    "# 1. Preparation\n",
    "We need to first import the required library, download the data, and load the data into the memory."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "swLpnk4u4XKo"
   },
   "source": [
    "## 1.1 Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 394,
     "output_extras": [
      {
       "item_id": 5
      },
      {
       "item_id": 6
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4509,
     "status": "ok",
     "timestamp": 1518570583550,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "9_NbEq2FSFLe",
    "outputId": "58bb96d9-3ffa-4ccf-b90a-c2db87f537b0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing required packages...\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\ChuanLi\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\ChuanLi\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "print('Importing required packages...')\n",
    "\n",
    "from IPython.display import clear_output\n",
    "import re\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "np.random.seed()\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "nltk.download('stopwords')\n",
    "from nltk.stem.wordnet import WordNetLemmatizer \n",
    "nltk.download('wordnet')\n",
    "from keras.preprocessing import sequence\n",
    "from keras.preprocessing import text as ktxt\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Input, Dense, Embedding, GRU, SpatialDropout1D, Bidirectional\n",
    "from keras.layers import concatenate, GlobalAveragePooling1D, GlobalMaxPooling1D\n",
    "from keras.layers.convolutional import Conv1D, MaxPooling1D\n",
    "from keras.models import Model\n",
    "from keras.callbacks import EarlyStopping\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "from sklearn.utils import class_weight\n",
    "\n",
    "\n",
    "def hint(message):\n",
    "    \"\"\"\n",
    "    erase previous ipynb output and show new message\n",
    "    \"\"\"\n",
    "    clear_output()\n",
    "    print(message)\n",
    "\n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0oMhFbcOS7ch"
   },
   "source": [
    "## 1.2 Loading the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 153,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 128,
     "status": "ok",
     "timestamp": 1518570590592,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "C1nwDbfiS-vX",
    "outputId": "d69716f3-d89c-49a1-91d3-7cfe42286b4b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label distribution between training and validation set:\n",
      "           label     train  validation\n",
      "0          toxic  0.096298    0.094031\n",
      "1   severe_toxic  0.009941    0.010215\n",
      "2        obscene  0.052790    0.053580\n",
      "3         threat  0.002992    0.003008\n",
      "4         insult  0.049351    0.049413\n",
      "5  identity_hate  0.008844    0.008648\n"
     ]
    }
   ],
   "source": [
    "hint('loading data...')\n",
    "train = pd.read_csv('data/train.csv')\n",
    "train, valid = train_test_split(train, test_size=0.2)\n",
    "\n",
    "labels = [\n",
    "    'toxic', \n",
    "    'severe_toxic', \n",
    "    'obscene', \n",
    "    'threat', \n",
    "    'insult', \n",
    "    'identity_hate'\n",
    "]\n",
    "\n",
    "Ytr = train[labels].values\n",
    "Yva = valid[labels].values\n",
    "\n",
    "hint('Label distribution between training and validation set:')\n",
    "print(pd.DataFrame({\n",
    "    'label': labels,\n",
    "    'train': [np.mean(train[label]) for label in labels],\n",
    "    'validation' : [np.mean(valid[label]) for label in labels],\n",
    "}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KA8OAJOJWAoe"
   },
   "source": [
    "# 2. Pre-processing the Input\n",
    "There are many ways to pre-process the raw strings into valid input for the model. Here we will do it by building a dictionary with all the comments from the training set, mapping the words to their index in the dictionary, and pad/crop the resulting sequences so that they have the same length."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZmEOAxm4wdNn"
   },
   "source": [
    "## 2.1 Cleaning Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 127,
     "status": "ok",
     "timestamp": 1518570683382,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "Q5jQUn7MxSud",
    "outputId": "73b337ab-1654-422d-cad2-a82158bd8880"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "tkzr = TweetTokenizer(preserve_case=False)\n",
    "eng_stopwords = (\n",
    "    'what', 'which', 'who', 'whom', \n",
    "    'this', 'that', 'these', 'those', \n",
    "    'am', 'is', 'are', 'was', 'were', \n",
    "    'be', 'been', 'being', \n",
    "    'have', 'has', 'had', 'having', \n",
    "    'do', 'does', 'did', 'doing', \n",
    "    'a', 'an', 'the', \n",
    "    'and', 'but', 'if', 'or', \n",
    "    'because', 'as', 'until', 'while', \n",
    "    'of', 'at', 'by', 'for', 'with', \n",
    "    'about', 'against', 'between', \n",
    "    'into', 'through', 'during', 'before', 'after', \n",
    "    'above', 'below', 'to', 'from', \n",
    "    'up', 'down', 'in', 'out', 'on', 'off', \n",
    "    'over', 'under', 'again', 'further', \n",
    "    'then', 'once', 'here', \n",
    "    'there', 'when', 'where', 'why', \n",
    "    'how', 'all', 'any', 'both', 'each', \n",
    "    'few', 'more', 'most', 'other', 'some', \n",
    "    'such', 'no', 'nor', 'not', 'only', \n",
    "    'own', 'same', 'so', 'than', 'too', 'very', \n",
    "    'can', 'will', 'just', 'don', 'should', 'now'\n",
    ")\n",
    "lmtzr = WordNetLemmatizer()\n",
    "appos = {\n",
    "    \"aren't\" : \"are not\",\n",
    "    \"can't\" : \"cannot\",\n",
    "    \"couldn't\" : \"could not\",\n",
    "    \"didn't\" : \"did not\",\n",
    "    \"doesn't\" : \"does not\",\n",
    "    \"don't\" : \"do not\",\n",
    "    \"hadn't\" : \"had not\",\n",
    "    \"hasn't\" : \"has not\",\n",
    "    \"haven't\" : \"have not\",\n",
    "    \"he'd\" : \"he would\",\n",
    "    \"he'll\" : \"he will\",\n",
    "    \"he's\" : \"he is\",\n",
    "    \"i'd\" : \"I would\",\n",
    "    \"i'd\" : \"I had\",\n",
    "    \"i'll\" : \"I will\",\n",
    "    \"i'm\" : \"I am\",\n",
    "    \"isn't\" : \"is not\",\n",
    "    \"it's\" : \"it is\",\n",
    "    \"it'll\":\"it will\",\n",
    "    \"i've\" : \"I have\",\n",
    "    \"let's\" : \"let us\",\n",
    "    \"mightn't\" : \"might not\",\n",
    "    \"mustn't\" : \"must not\",\n",
    "    \"shan't\" : \"shall not\",\n",
    "    \"she'd\" : \"she would\",\n",
    "    \"she'll\" : \"she will\",\n",
    "    \"she's\" : \"she is\",\n",
    "    \"shouldn't\" : \"should not\",\n",
    "    \"that's\" : \"that is\",\n",
    "    \"there's\" : \"there is\",\n",
    "    \"they'd\" : \"they would\",\n",
    "    \"they'll\" : \"they will\",\n",
    "    \"they're\" : \"they are\",\n",
    "    \"they've\" : \"they have\",\n",
    "    \"we'd\" : \"we would\",\n",
    "    \"we're\" : \"we are\",\n",
    "    \"weren't\" : \"were not\",\n",
    "    \"we've\" : \"we have\",\n",
    "    \"what'll\" : \"what will\",\n",
    "    \"what're\" : \"what are\",\n",
    "    \"what's\" : \"what is\",\n",
    "    \"what've\" : \"what have\",\n",
    "    \"where's\" : \"where is\",\n",
    "    \"who'd\" : \"who would\",\n",
    "    \"who'll\" : \"who will\",\n",
    "    \"who're\" : \"who are\",\n",
    "    \"who's\" : \"who is\",\n",
    "    \"who've\" : \"who have\",\n",
    "    \"won't\" : \"will not\",\n",
    "    \"wouldn't\" : \"would not\",\n",
    "    \"you'd\" : \"you would\",\n",
    "    \"you'll\" : \"you will\",\n",
    "    \"you're\" : \"you are\",\n",
    "    \"you've\" : \"you have\",\n",
    "    \"'re\": \" are\",\n",
    "    \"wasn't\": \"was not\",\n",
    "    \"we'll\":\" will\",\n",
    "    \"didn't\": \"did not\"\n",
    "}\n",
    "\n",
    "def preprocess(comment):\n",
    "  \n",
    "    # credit to the author of this post:\n",
    "    # https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda\n",
    "\n",
    "    # remove special format\n",
    "    comment = re.sub('\\n\\t', '', comment)\n",
    "\n",
    "    # remove IP addresses\n",
    "    comment = re.sub('\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}', ' specipaddress ', comment)\n",
    "\n",
    "    # remove username\n",
    "    comment = re.sub(\"\\[\\[User.*\\]\", ' specusername ', comment)\n",
    "    comment = re.sub(\"\\[\\[User.*\\|\", ' specusername ', comment)\n",
    "\n",
    "    # tokenization \n",
    "    tokens = tkzr.tokenize(comment)\n",
    "\n",
    "    # aphostophe replacement\n",
    "    tokens = [ appos[token] if token in appos else token for token in tokens]\n",
    "\n",
    "    # remove stopwords\n",
    "    tokens = [ token for token in tokens if not token in eng_stopwords ]\n",
    "\n",
    "    # stemming\n",
    "    tokens = [ lmtzr.lemmatize(token, 'v') for token in tokens]\n",
    "\n",
    "    return \" \".join(tokens)\n",
    "  \n",
    "\n",
    "hint('Cleaning train set...')\n",
    "Xtr = train['comment_text'].apply(lambda c: preprocess(c))\n",
    "hint('Cleaning test set...')\n",
    "Xva = valid['comment_text'].apply(lambda c: preprocess(c))\n",
    "hint('Done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "btLphIdFwmts"
   },
   "source": [
    "## 2.2 Transforming Comments to Sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 144,
     "status": "ok",
     "timestamp": 1518570698059,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "JmEQQZs-kHK2",
    "outputId": "073af561-39f5-4c18-acee-ab2794e9e3a6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "vocab_max = 100000\n",
    "\n",
    "hint('Fitting the tokenizer...')\n",
    "tokenizer = ktxt.Tokenizer(num_words=vocab_max)\n",
    "tokenizer.fit_on_texts(Xtr)\n",
    "\n",
    "hint('Tokenizing...')\n",
    "Xtr = tokenizer.texts_to_sequences(Xtr)\n",
    "Xva = tokenizer.texts_to_sequences(Xva)\n",
    "\n",
    "hint('padding the sequences...')\n",
    "max_comment_length = 200  # padded/cropped comment length\n",
    "Xtr = sequence.pad_sequences(Xtr, maxlen=max_comment_length)\n",
    "Xva = sequence.pad_sequences(Xva, maxlen=max_comment_length)\n",
    "\n",
    "hint('Done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "OWHeGxuz3Wfn"
   },
   "source": [
    "# 3. Training Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "hint(\"Loading pre-embedding file...\")\n",
    "emb = pd.read_table(\n",
    "    'preembedding/glove.6B.300d.txt', \n",
    "    sep=\" \", index_col=0, header=None, quoting=csv.QUOTE_NONE\n",
    ")\n",
    "\n",
    "hint(\"Preparing embedding matrix...\")\n",
    "embedding_dimension = 300\n",
    "embedding_matrix = np.random.normal(\n",
    "    emb.mean(axis=0), \n",
    "    emb.std(axis=0), \n",
    "    (vocab_max, embedding_dimension)\n",
    ")\n",
    "hint(\"Constructing embedding matrix\")\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    if i < vocab_max and word in emb.index:\n",
    "        embedding_matrix[i] = emb.loc[word].as_matrix()\n",
    "\n",
    "hint(\"Done\")\n",
    "# optional: free memory:\n",
    "emb = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 374,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 389,
     "status": "ok",
     "timestamp": 1518570698678,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "GoawhjBA3nyc",
    "outputId": "e814cec6-a98e-4cce-da53-d2579963c70f"
   },
   "outputs": [],
   "source": [
    "model = None\n",
    "x = None\n",
    "sequence_input = Input(shape=(max_comment_length, ))\n",
    "x = Embedding(vocab_max, embedding_dimension, weights=[embedding_matrix])(sequence_input)\n",
    "x = SpatialDropout1D(0.5)(x)\n",
    "x = Conv1D(filters=256, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = MaxPooling1D(pool_size=2)(x)\n",
    "x = SpatialDropout1D(0.2)(x)\n",
    "x = Bidirectional(GRU(64, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "avg_pool = GlobalAveragePooling1D()(x)\n",
    "max_pool = GlobalMaxPooling1D()(x)\n",
    "x = concatenate([avg_pool, max_pool])\n",
    "preds = Dense(6, activation=\"sigmoid\")(x)\n",
    "model = Model(sequence_input, preds)\n",
    "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1S6kDCqDFGUN"
   },
   "source": [
    "Now training the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 122,
     "output_extras": [
      {
       "item_id": 373
      },
      {
       "item_id": 1155
      },
      {
       "item_id": 1418
      },
      {
       "item_id": 2339
      },
      {
       "item_id": 2340
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 630277,
     "status": "ok",
     "timestamp": 1518571329571,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "QVU9nuDzFtlf",
    "outputId": "e0ea2b97-fe62-42bf-a1b2-46c7d5f08532"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/4\n",
      "127656/127656 [==============================] - 332s 3ms/step - loss: 0.1506 - acc: 0.9787 - val_loss: 0.0454 - val_acc: 0.9833\n",
      "Epoch 2/4\n",
      "127656/127656 [==============================] - 328s 3ms/step - loss: 0.1117 - acc: 0.9829 - val_loss: 0.0431 - val_acc: 0.9838\n",
      "Epoch 3/4\n",
      "127656/127656 [==============================] - 326s 3ms/step - loss: 0.0999 - acc: 0.9842 - val_loss: 0.0434 - val_acc: 0.9839\n",
      "Epoch 4/4\n",
      "127656/127656 [==============================] - 327s 3ms/step - loss: 0.0921 - acc: 0.9851 - val_loss: 0.0429 - val_acc: 0.9834\n"
     ]
    }
   ],
   "source": [
    "epochs = 4\n",
    "batch_size = 64\n",
    "\n",
    "def get_class_weight(x):\n",
    "    k = 100\n",
    "    return 3.32*np.log(k/x + 1)\n",
    "    \n",
    "\n",
    "history = model.fit(\n",
    "    Xtr, Ytr, \n",
    "    epochs=epochs, \n",
    "    batch_size=batch_size,\n",
    "    validation_data=(Xva, Yva),\n",
    "    class_weight={\n",
    "        0: get_class_weight(98),\n",
    "        1: get_class_weight(10),\n",
    "        2: get_class_weight(53),\n",
    "        3: get_class_weight(2),\n",
    "        4: get_class_weight(49),\n",
    "        5: get_class_weight(8),\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nAFiq9QuFI7R"
   },
   "source": [
    "Making prediction on the validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 121,
     "status": "ok",
     "timestamp": 1518571356080,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "hgiIMLjZE3Rr",
    "outputId": "f00c0add-a7ab-43f7-b75d-c47a54e12890"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "hint(\"Making prediction...\")\n",
    "Yva_ = model.predict(Xva)\n",
    "hint(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RfRNkLv7v04I"
   },
   "source": [
    "# 4. Result Analysis\n",
    "## 4.1 Global Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 255,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 275,
     "status": "ok",
     "timestamp": 1518571356419,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "vakvDUO31-oo",
    "outputId": "5b7acf10-a987-41f7-801c-0f291245b687"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation set sample count: 31915\n",
      "\n",
      "accuracy for threshold 0.1: 96.92%\n",
      "accuracy for threshold 0.2: 97.64%\n",
      "accuracy for threshold 0.3: 98.01%\n",
      "accuracy for threshold 0.4: 98.21%\n",
      "accuracy for threshold 0.5: 98.34%\n",
      "accuracy for threshold 0.6: 98.41%\n",
      "accuracy for threshold 0.7: 98.39%\n",
      "accuracy for threshold 0.8: 98.30%\n",
      "accuracy for threshold 0.9: 98.07%\n",
      "\n",
      "best threshold: 0.6\n",
      "best accuracy: 98.41%\n"
     ]
    }
   ],
   "source": [
    "total_sample = Xva.shape[0]\n",
    "print(\"validation set sample count: %d\\n\" % total_sample)\n",
    "prediction_total = total_sample*Yva.shape[1]\n",
    "best_t = None\n",
    "best_accuracy = 0\n",
    "for t in [i*0.1 for i in range(1, 10)]:\n",
    "    accuracy = np.sum(Yva == (Yva_ >= t))/prediction_total\n",
    "    if accuracy > best_accuracy: \n",
    "        best_t = t\n",
    "        best_accuracy = accuracy\n",
    "    print(\"accuracy for threshold %.1f: %.2f%%\" % (t, accuracy*100))\n",
    "Yva_T = Yva_ >= best_t\n",
    "correct = Yva == Yva_T\n",
    "print(\"\\nbest threshold: %.1f\" % best_t)\n",
    "print(\"best accuracy: %.2f%%\" % (best_accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YSptfDNc58dZ"
   },
   "source": [
    "## 4.2 Accuracy by Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "gLRH9AayowXA"
   },
   "outputs": [],
   "source": [
    "overview = pd.DataFrame(index=[\n",
    "    'label‰ of all',\n",
    "    'total wrong', \n",
    "    'P->N', \n",
    "    'N->P', \n",
    "    'P->N %', \n",
    "    'N->P %',\n",
    "    'avg len',\n",
    "])\n",
    "\n",
    "def analyze_class(i):\n",
    "    wrong = valid[correct[:, i] != 1]\n",
    "    total_class_error = len(wrong)\n",
    "    print(\"%d predicted incorrectly (%.2f%% of all samples)\" % (\n",
    "        total_class_error, \n",
    "        100*total_class_error/total_sample\n",
    "    ))\n",
    "        \n",
    "    wrong_seqs = Xva[correct[:, i] != 1]\n",
    "    lens = [ len(seq[seq != 0]) for seq in wrong_seqs]\n",
    "    avg_len = np.mean(lens)\n",
    "    print(\"Falsely predicted sequences have an average length of %d\" % avg_len)\n",
    "\n",
    "    PpN = valid[(Yva[:, i] == 1) & (Yva_T[:, i] == 0)]\n",
    "    PpN_count = len(PpN)\n",
    "    print(\"\\n%d (%.2f%%) positive label were predicted to be negative\" % (\n",
    "        PpN_count, \n",
    "        100*PpN_count/total_class_error \n",
    "    ))\n",
    "    if PpN_count > 4:\n",
    "        print(\"Samples:\")\n",
    "        for sample in PpN.sample(5)['comment_text']:\n",
    "            display(sample)\n",
    "  \n",
    "    NpP = valid[(Yva[:, i] == 0) & (Yva_T[:, i] == 1)]\n",
    "    NpP_count = len(NpP)\n",
    "    print(\"\\n%d (%.2f%%) negative label were predicted to be positive\" % (\n",
    "        NpP_count, \n",
    "        100*NpP_count/total_class_error \n",
    "    ))\n",
    "    if NpP_count > 4:\n",
    "        print(\"Samples:\")\n",
    "        for sample in NpP.sample(5)['comment_text']:\n",
    "            display(sample)\n",
    "  \n",
    "    overview[labels[i]] = [\n",
    "        np.mean(Yva[:, i]*1000),\n",
    "        total_class_error, \n",
    "        PpN_count,  \n",
    "        NpP_count,\n",
    "        100*PpN_count/total_class_error,\n",
    "        100*NpP_count/total_class_error,\n",
    "        avg_len\n",
    "    ]\n",
    "  \n",
    "    print('\\n')\n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SSY9H7knpE5U"
   },
   "source": [
    "### 4.2.1 Toxic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 360,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      },
      {
       "item_id": 8
      },
      {
       "item_id": 9
      },
      {
       "item_id": 10
      },
      {
       "item_id": 11
      },
      {
       "item_id": 12
      },
      {
       "item_id": 13
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 251,
     "status": "ok",
     "timestamp": 1518573461778,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "NNGw690DpLMy",
    "outputId": "18e4b55e-63d1-42b2-9c93-4b75a8d5ad39"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1085 predicted incorrectly (3.40% of all samples)\n",
      "Falsely predicted sequences have an average length of 33\n",
      "\n",
      "635 (58.53%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"SERIOUSLY \\n\\nAS SOON ASI'M UNBANENED I WLL WREACFK EVERYTHING\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\n Your claims of \"\"weasel wording\"\" are lies and attempts to poison the well. -  ✉ \"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'YOUR PAGE \\n\\nI own this page, if you dont comply I will sue you for 1 billion dollars'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'i dare you \\n\\nBlock me. I will do it again, i dare you to reply to my discussions rather owning articles and issuing warnings.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Halt! You appear to have joined Wikipedia to promote your Racist Beliefs. This is not the place for old men with high school educations to vent their rage at minorities, friend.   \\n\\nWikipedia is not a soapbox to promote your racist beliefs, bigot. You know about as much about History as the Pope knows about intercourse. Please take your tired, uniformed bigotry somewhere else. Why not take a visit to the local chapter of the Ku Klux Klan? Your beliefs will fit in much better there. Thanks.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "450 (41.47%) negative label were predicted to be positive\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Hey loser get a life.  Time to reset IP woot.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'AGAIN YOU SLANDER HIS NAME! \\n\\nI get strange message that I cannot attack user and must attack content! I am attacking the content, the content of a wrong that was unjustly delivered upon one of my Brotherhood members. LEITMOTIV.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"You reverted my edition when I added Arouca but when SLBedit added it you didn't! Wtf?\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Fun \\n\\nWikipedia can be edited by anyone and your ruining how a random person can make up rubbish on the pages which idiots might just believe. Also how do you know what Frankie Boyle has done, or what Arbroath is like?'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\n\\n Who is Dreadstar and How is it that they are able to censor me? \\n\\nSo I log into Wikipedia one evening and find a few things worth commenting on.  Suddenly all my edits are blocked by \"\"Dreadstar.\"\"  Who is this censor?  Why does he have the right to determine that what my opinions about particular edits on archeological topics are vandalism?\\n\\nWTF and who the heck is Dreadstar?\"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3cxSNKFepEg1"
   },
   "source": [
    "### 4.2.2 Severe Toxic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 360,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      },
      {
       "item_id": 8
      },
      {
       "item_id": 9
      },
      {
       "item_id": 10
      },
      {
       "item_id": 11
      },
      {
       "item_id": 12
      },
      {
       "item_id": 13
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 254,
     "status": "ok",
     "timestamp": 1518573463811,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "1cCGMbZKpV1H",
    "outputId": "428d69ea-64a9-4f28-e09c-f9ddd6d7516c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "318 predicted incorrectly (1.00% of all samples)\n",
      "Falsely predicted sequences have an average length of 31\n",
      "\n",
      "228 (71.70%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'fuck you shut the fuck up'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\n\\n FUCK WIKIPEDIA \\n\\nFUCK THIS WEBSITE MAN, I TRY AND TRY TO GIVE GOOD INFORMATION TO THIS SHITTY WEBSITE, AND ALL I GET IS SHIT SAYING \"\"FOLLOW WIKIPEDIA GUILDLINES BLA BLA BLA BLA\"\" FUCK WIKIPEDIA GUIDELINES, I DESEVERE SOME RESPECT FOR WHAT I DO, AND THIS IS A WARNING, IF I DO NOT GET AN APOLOGE IN THE NEXT 48 HOURS, THEN I\\'LL GET THIS FUCKING WEBSITE SHUT DOWN FOR GOOD.\\n\\nSENCERILY, MY ASS\"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"Go fuck yourself this ain't any of your business and i fucking know what tor is. Asshole.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Your a dick head!! \\n\\nHow dare you go around deleting peoples articles. What kind of retard spends all their time on the computer... you, you nerd!!'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'GET A LIFE shit Nerd!'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "90 (28.30%) negative label were predicted to be positive\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'scum who look up to Che deserve the worst kind of death.==\\n\\nyou faggot little bitches suck balls.\\n\\n=='"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"YOU FUCKING NIGGER, HOW DARE YOU REVERT MY EDITS???? I'LL HUNT YOU DOWN AND RAPE YOU UNTIL YOU SHIT YOUR INTERNAL ORGANS OUT\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Fuck you niggerkite. y you deleting so much shit. fuckn loser. dont delete anymore shit. its none of your fucking business'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'fuck u jimmy wales fuck fuck fuck'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "') \\n\\nFuck you. Eat shit and die.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "s0_zIe7m1A79"
   },
   "source": [
    "### 4.2.3 Obscene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 360,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      },
      {
       "item_id": 8
      },
      {
       "item_id": 9
      },
      {
       "item_id": 10
      },
      {
       "item_id": 11
      },
      {
       "item_id": 12
      },
      {
       "item_id": 13
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 258,
     "status": "ok",
     "timestamp": 1518573465290,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "Qt9vOoNK1EFl",
    "outputId": "cfeec26f-71ad-4668-840e-c183a862ba74"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "563 predicted incorrectly (1.76% of all samples)\n",
      "Falsely predicted sequences have an average length of 37\n",
      "\n",
      "264 (46.89%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'One pint of feces. \\n\\nON YOUR FAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACE'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'B-5? \\n\\nExcuse me but what do you mean by contributing in a profeshionall level of bullshit?\\nUser 8th sinn'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'How do you know he is dead.  Its just his plane that crashed.  Jeezz, quit busting his nuts, folks.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'stupid\\nYes he did JBKramer called me stupid, here  in an edit summary.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'I wanted to let you know that Tymon.r. is a stupid moron.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "299 (53.11%) negative label were predicted to be positive\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"So, making false accusation of vandalism, deliberately inserting false information into articles, reverting for no reason and harassment is just fine, no need for even so much as a warning.  Meanwhile removing false information from articles merits a block.  What a fucked up place this is.  Well, see you on the next IP shortly!  You will not win this battle.  You will not prevent me from improving articles.  You will not be able to keep wind wave or any of the articles I've improved in a deficient state.  So why even try?  190.161.186.94\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"Giant oarfish edits \\n\\nHi, instead of completely reverting my edit like a lazy prick, perhaps you should just remove my commentary, instead of being a lazy prick. Did I mention that you're a lazy prick? Oh, and you're a lazy prick.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'FAG FAG FAG FAG FAG FAG FAG... GET A REAL JOB PANSY!'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\n\\n Response to HJ Mitchells \"\"challenge\"\" \\n\\nSince I am still blocked and cannot post this in response to HJ\\'s challenge nor can I vote for the three candidates I do support, I will leave this here.\\nHj Mitchell, please spare us the dramatics of, \"\"I\\'ll quite if you give me a reason.\"\" No one here thinks you will quite regardless of any reasons given nor would you be expected too. You are also fully aware that no one is going to mine through your edits looking for wrongdoing. That\\'s just a waste of time and you would likely retort with they are NOTHERE. What I will do, is give you a few examples of what I have seen over a very short span of time. Given a review of any say, month long period, I think we will find the same result within a few percentage points of the mean.\\nYou made an obviously bad call about Protonk, a well respected and long term contributor to this project based on what you term a BLP violation. You state in your comments that this was supported by other admins when anyone who takes the time to actually go there and read the comments will see, that no one supported your decision. This caused a moderate admin to leave the project completely only to allow less moderate admins like yourself more room to exert influence. After reviewing the comment Protonk made, I certainly agree it was a bad comment, but it was absolutely not worthy of a block without any sort of warning or talk page message and it was no worse than your outburst of telling all the editors in an RFA they are Fucking morons\"\" because you were the submitter of the RFA and likely had your feelings hurt. An event I note that you were not blocked nor even approached about and as far I can find have never openly apologized for. Although there may be a comment somewhere I do admit. \\nYou have a tendency to exaggerate in your favor such as can be seen in the previous statement when determining that there was support to summarily block Protonk when there clearly was nt. This can also be seen in reviewing some of the cases on your talk page and talk archives.\\nAs mentioned above, and as noted by several individuals, your Fucking morons\\'\\'\\' comment is still too new and does not represent the attitude needed or wanted of a functionary.\\nYou recently unblocked  and revoked his topic ban knowing this user had a long history of Personal attacks and trolling regarding gender topics and gamergate. Within 24 hours that user was again blocked for personal attacks, has continued to stir the pot and make drama and has been drug to at least ANI and Arbcom within the last week as can be seen on their talk page and the mentioned venues.\\nEven based on your comments here, you clearly have an attitude of knowing better than everyone else whats good for Wikipedia. That arrogance is inappropriate of a functionary.\\nYou give a staggering number of over 13000 blocks and claim it as an achievement. Personally, I see this as proof of your tendency to block first and ask questions later. There is no way you scrutinized all those and I can easily see a lot of mistakes and preventing useful edits from being done through bad faith assumptions of guilt, similar to those you displayed towards me for criticizing your actions.\\nI could keep going, but that is enough IMO to show a pattern of conduct not appropriate of a functionary. 96.255.237.170  \"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'YOU WHATS THE DEAL U PICE OF COW CRAP \\n\\nu CAN EAT MY BALLZ'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Lwpco7aB1GrZ"
   },
   "source": [
    "### 4.2.4 Threat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 258,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 257,
     "status": "ok",
     "timestamp": 1518573466922,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "gQgcRgOq1GLO",
    "outputId": "69893ed6-1588-41db-eff3-e86766fac8e6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93 predicted incorrectly (0.29% of all samples)\n",
      "Falsely predicted sequences have an average length of 27\n",
      "\n",
      "81 (87.10%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'FUCK YOU \\n\\nYOU ARE THE BIGGEST PIG ON THE PLANET.\\n\\nDIE IN HELL ASSHOLE!'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'YOU WILL DIE IN 7 DAYS.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Go fuck yourself \\n\\nEat shit and die. 68.0.119.139'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'How to kill mozart \\n\\nWOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO STAB HIM, SHOOT HIM, JUMP ON HIM'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Die \\n\\nI HATE YOU PRICK YOU DINT DESERVE A PLACE HERE'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "12 (12.90%) negative label were predicted to be positive\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"Just be quiet. \\n\\nWho the fuck are you to tell me what to do? I will post what I want, where I want, and if you don't like it, you can get cancer and die.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE RAPE'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"cardiac arrest and died.\"\"\\n\\n\"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Hi! \\n\\nI wanna rape you!'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'If a girl ever sees you naked, she will cry for laughing so hard. \\n\\nYou should just kill yourself.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6lkRScLJ1fGN"
   },
   "source": [
    "### 4.2.5 Insult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 360,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      },
      {
       "item_id": 8
      },
      {
       "item_id": 9
      },
      {
       "item_id": 10
      },
      {
       "item_id": 11
      },
      {
       "item_id": 12
      },
      {
       "item_id": 13
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 270,
     "status": "ok",
     "timestamp": 1518573480301,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "zJ2VKxgK1hRC",
    "outputId": "68a3f353-831b-426f-85dc-9d65a39561cd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "749 predicted incorrectly (2.35% of all samples)\n",
      "Falsely predicted sequences have an average length of 31\n",
      "\n",
      "450 (60.08%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Get out my talk page you smelling wog!'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"You're right. I don't like how Wikipedia works, I don't like any of the experienced editors. I shouldn't be here. I did not insult anyone because I think I'm better. I do not want a Wikipedia page for your previously stated reasons. I use Wikipedia and consider it credible.nif anyone can tell you how big my dick is, it's me. Neither you nor your website give me any clout. I wasn't promoting. What am I going to say, hey go check out my Wikipedia. Fuck you, your editor friends and Wikipedia. And fuck my Wikipedia page I dont want one   \\n\\nadd that to my wiki \\n\\nI hope they ban me and delete my account so I don't have to do it\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Why not admit it? The only reason you disagree with anything is because you are an obsessed troll, whose sole agenda is causing discord.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\"\\n\\nHello, you recently yipped and yapped about me breaking the rules, but I notice that your latest update you edited in \"\"a proud faggot nutsack\"\" within my userpage.  I find this intolerable, bigoted, and downright hateful.  Perhaps homosexual African-Americans elecit anger in you?  You should visit www.adl.org .  \"'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Lack of New York Post report in the artical  \\n\\n lack of Mark Lavin / New York Post report in the artical  \\n There should be someting in the artical about Mark Lavin exposing Olbermann about his alegedly being a pervert with a small penis audio here Artical Here'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "299 (39.92%) negative label were predicted to be positive\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"Arrgh \\n\\nOK I'll go back to wikiHow it's fun unlike dumb Wikiepedia it sucks it's boring I hate it goodbye block me fucker I don't give a shit!!\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Quit your snivelling, you twit. There is nothing you can do stop me from fixing this mess that some refer to as an encyclopedia.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'I have to log in moron to reply, this is my normal ip jerk and you made countless more mistakes, not honest!'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"You're a douchebag \\n\\nIf you go back 24 hours and think about things, you would realize that blocking me from editing for 24 hours was a stupid decision. \\n\\nFanovian gave me my final warning for incivility and vandalism on the page pertaining to the nonexistent lord and savior Jesus. after this warning I stopped hurting everyones feelings. Just 5 seconds later you blocked me. \\n\\nNow either you are a complete fucking twat, or you simply cannot read, either way I should not have been blocked for doing nothing which I could have been blocked for. \\n\\nNo hard feelings though, I know you didn't ask to be mentally retarded, and you can't help the fact that you are incapacitated. \\n\\nMuch love,\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'DOO DOO DA DA DA DA  GO SCREW YOURSELF'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ybjmae651jxY"
   },
   "source": [
    "### 4.2.6 Identity Hate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 258,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 3
      },
      {
       "item_id": 4
      },
      {
       "item_id": 5
      },
      {
       "item_id": 6
      },
      {
       "item_id": 7
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 440,
     "status": "ok",
     "timestamp": 1518573482230,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "6ojSnXP51jPm",
    "outputId": "d1ee8ad8-5407-43c8-cab2-a5f77112fa2c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "237 predicted incorrectly (0.74% of all samples)\n",
      "Falsely predicted sequences have an average length of 33\n",
      "\n",
      "225 (94.94%) positive label were predicted to be negative\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'your mums ugly \\n\\nyour gay'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Hi Motherfucker Zhanzhao, are you a slant-eyed chink or a blackie indian or both? \\n\\nSays the above. Answer my question, Fucker. 218.186.12.250'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\"Busy in real life?! LOL, pathetic cunt. Learn to read English, while you have time, fucking piece of chingching trash. You've to see who was right, poor squint. But yeah, you Asian animals don't have self-judgement.\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'PENIS VAGINA YOU SUCK FAGGOT ASSHOLES AT WIKIPEDIA'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'RSPW Poster is a sick gay freak'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "12 (5.06%) negative label were predicted to be positive\n",
      "Samples:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'YOU ARE A MINDLESS FAGGOT PIG PLEASE GOD BURN THIS FUCKING MAGGOT TO DEATH IN HELL\\nYOU HAVE OFFENDED MY SENSES YOU MINDLESS SELF IMPORTANT FUCKING PEST\\nTHE GOD THAT YOU HATE HATES YOU'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'ura fag... \\n\\nu gay L'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Hello fuckhead,\\nContributions my faggot asshole. You think that because im gay that i dont know shit. you are wrong. you just sit there with nothing to do with your pathetic fucking life that you decide to go and be a complete prick on wikipedia and just delete whatever the fuck u want. Thanks nazi fuck.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "', what ever you do. Do not be gay'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'YOU LIKE COCK 8==========D - - - - - \\n\\nNIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER NIGGER .'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "analyze_class(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GYOBKlWc12Rc"
   },
   "source": [
    "### 4.2.7 Overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 235,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 247,
     "status": "ok",
     "timestamp": 1518573484262,
     "user": {
      "displayName": "Chuan Li",
      "photoUrl": "//lh3.googleusercontent.com/-ry1V_A03qTg/AAAAAAAAAAI/AAAAAAAAAh4/Ea41Em8n49A/s50-c-k-no/photo.jpg",
      "userId": "106476928809257656982"
     },
     "user_tz": 480
    },
    "id": "w9g39VJ4n00f",
    "outputId": "8e4f9efb-795c-4253-de18-d7b2be471bb0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>label‰ of all</th>\n",
       "      <td>94</td>\n",
       "      <td>10</td>\n",
       "      <td>53</td>\n",
       "      <td>3</td>\n",
       "      <td>49</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total wrong</th>\n",
       "      <td>1085</td>\n",
       "      <td>318</td>\n",
       "      <td>563</td>\n",
       "      <td>93</td>\n",
       "      <td>749</td>\n",
       "      <td>237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P-&gt;N</th>\n",
       "      <td>635</td>\n",
       "      <td>228</td>\n",
       "      <td>264</td>\n",
       "      <td>81</td>\n",
       "      <td>450</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>N-&gt;P</th>\n",
       "      <td>450</td>\n",
       "      <td>90</td>\n",
       "      <td>299</td>\n",
       "      <td>12</td>\n",
       "      <td>299</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P-&gt;N %</th>\n",
       "      <td>58</td>\n",
       "      <td>71</td>\n",
       "      <td>46</td>\n",
       "      <td>87</td>\n",
       "      <td>60</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>N-&gt;P %</th>\n",
       "      <td>41</td>\n",
       "      <td>28</td>\n",
       "      <td>53</td>\n",
       "      <td>12</td>\n",
       "      <td>39</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg len</th>\n",
       "      <td>33</td>\n",
       "      <td>31</td>\n",
       "      <td>37</td>\n",
       "      <td>27</td>\n",
       "      <td>31</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               toxic  severe_toxic  obscene  threat  insult  identity_hate\n",
       "label‰ of all     94            10       53       3      49              8\n",
       "total wrong     1085           318      563      93     749            237\n",
       "P->N             635           228      264      81     450            225\n",
       "N->P             450            90      299      12     299             12\n",
       "P->N %            58            71       46      87      60             94\n",
       "N->P %            41            28       53      12      39              5\n",
       "avg len           33            31       37      27      31             33"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overview.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "default_view": {},
   "name": "RCNN performance analysis.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}