Multi RCNN Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we will find the optimal learing rate for the pooled RCNN model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "\n",
    "def hint(message):\n",
    "    \"\"\"\n",
    "    erase previous ipynb output and show new message\n",
    "    \"\"\"\n",
    "    clear_output()\n",
    "    print(message)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label distribution between training and validation set:\n",
      "           label     train  validation\n",
      "0          toxic  0.096564    0.092966\n",
      "1   severe_toxic  0.010270    0.008899\n",
      "2        obscene  0.053166    0.052076\n",
      "3         threat  0.003032    0.002851\n",
      "4         insult  0.049798    0.047627\n",
      "5  identity_hate  0.008750    0.009024\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "hint('loading data...')\n",
    "train = pd.read_csv('data/train.csv')\n",
    "train, valid = train_test_split(train, test_size=0.2)\n",
    "\n",
    "labels = [\n",
    "    'toxic', \n",
    "    'severe_toxic', \n",
    "    'obscene', \n",
    "    'threat', \n",
    "    'insult', \n",
    "    'identity_hate'\n",
    "]\n",
    "\n",
    "Ytr = train[labels].values\n",
    "Yva = valid[labels].values\n",
    "\n",
    "hint('Label distribution between training and validation set:')\n",
    "class_proportion = pd.DataFrame({\n",
    "    'label': labels,\n",
    "    'train': [np.mean(train[label]) for label in labels],\n",
    "    'validation' : [np.mean(valid[label]) for label in labels],\n",
    "})\n",
    "print(class_proportion)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Pre-processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Gather resources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\ChuanLi\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "import nltk\n",
    "\n",
    "nltk.download('wordnet')\n",
    "\n",
    "lmtzr = WordNetLemmatizer()\n",
    "eng_stopwords = (\n",
    "    'what', 'which', 'who', 'whom', \n",
    "    'this', 'that', 'these', 'those', \n",
    "    'am', 'is', 'are', 'was', 'were', \n",
    "    'be', 'been', 'being', \n",
    "    'have', 'has', 'had', 'having', \n",
    "    'do', 'does', 'did', 'doing', \n",
    "    'a', 'an', 'the', \n",
    "    'and', 'but', 'if', 'or', \n",
    "    'because', 'as', 'until', 'while', \n",
    "    'of', 'at', 'by', 'for', 'with', \n",
    "    'about', 'against', 'between', \n",
    "    'into', 'through', 'during', 'before', 'after', \n",
    "    'above', 'below', 'to', 'from', \n",
    "    'up', 'down', 'in', 'out', 'on', \n",
    "    'over', 'under', 'again', 'further', \n",
    "    'then', 'once', 'here', \n",
    "    'there', 'when', 'where', 'why', \n",
    "    'how', 'all', 'any', 'both', 'each', \n",
    "    'few', 'more', 'most', 'other', 'some', \n",
    "    'such', 'nor', 'only', \n",
    "    'own', 'same', 'so', 'than', 'too', 'very', \n",
    "    'can', 'will', 'just', 'don', 'should', 'now'\n",
    ")\n",
    "appos = {\n",
    "    \"aren't\" : \"are not\",\n",
    "    \"can't\" : \"cannot\",\n",
    "    \"couldn't\" : \"could not\",\n",
    "    \"didn't\" : \"did not\",\n",
    "    \"doesn't\" : \"does not\",\n",
    "    \"don't\" : \"do not\",\n",
    "    \"hadn't\" : \"had not\",\n",
    "    \"hasn't\" : \"has not\",\n",
    "    \"haven't\" : \"have not\",\n",
    "    \"he'd\" : \"he would\",\n",
    "    \"he'll\" : \"he will\",\n",
    "    \"he's\" : \"he is\",\n",
    "    \"i'd\" : \"I would\",\n",
    "    \"i'd\" : \"I had\",\n",
    "    \"i'll\" : \"I will\",\n",
    "    \"i'm\" : \"I am\",\n",
    "    \"isn't\" : \"is not\",\n",
    "    \"it's\" : \"it is\",\n",
    "    \"it'll\":\"it will\",\n",
    "    \"i've\" : \"I have\",\n",
    "    \"let's\" : \"let us\",\n",
    "    \"mightn't\" : \"might not\",\n",
    "    \"mustn't\" : \"must not\",\n",
    "    \"shan't\" : \"shall not\",\n",
    "    \"she'd\" : \"she would\",\n",
    "    \"she'll\" : \"she will\",\n",
    "    \"she's\" : \"she is\",\n",
    "    \"shouldn't\" : \"should not\",\n",
    "    \"that's\" : \"that is\",\n",
    "    \"there's\" : \"there is\",\n",
    "    \"they'd\" : \"they would\",\n",
    "    \"they'll\" : \"they will\",\n",
    "    \"they're\" : \"they are\",\n",
    "    \"they've\" : \"they have\",\n",
    "    \"we'd\" : \"we would\",\n",
    "    \"we're\" : \"we are\",\n",
    "    \"weren't\" : \"were not\",\n",
    "    \"we've\" : \"we have\",\n",
    "    \"what'll\" : \"what will\",\n",
    "    \"what're\" : \"what are\",\n",
    "    \"what's\" : \"what is\",\n",
    "    \"what've\" : \"what have\",\n",
    "    \"where's\" : \"where is\",\n",
    "    \"who'd\" : \"who would\",\n",
    "    \"who'll\" : \"who will\",\n",
    "    \"who're\" : \"who are\",\n",
    "    \"who's\" : \"who is\",\n",
    "    \"who've\" : \"who have\",\n",
    "    \"won't\" : \"will not\",\n",
    "    \"wouldn't\" : \"would not\",\n",
    "    \"you'd\" : \"you would\",\n",
    "    \"you'll\" : \"you will\",\n",
    "    \"you're\" : \"you are\",\n",
    "    \"you've\" : \"you have\",\n",
    "    \"'re\": \" are\",\n",
    "    \"wasn't\": \"was not\",\n",
    "    \"we'll\":\" will\",\n",
    "    \"didn't\": \"did not\"\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define cleaning method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import TweetTokenizer\n",
    "import re\n",
    "\n",
    "tkzr = TweetTokenizer(preserve_case=False)\n",
    "\n",
    "def preprocess(comment):\n",
    "  \n",
    "    # credit to the author of this post:\n",
    "    # https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda\n",
    "    \n",
    "    # remove urls\n",
    "    comment = re.sub(r'https?://en.wikipedia.org/[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]', ' speclinkaddress ', comment)\n",
    "    comment = re.sub(r'https?://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]', ' speclinkaddress ', comment)\n",
    "    comment = re.sub(r'www.[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]', ' speclinkaddress ', comment)\n",
    "\n",
    "    # remove IP addresses\n",
    "    comment = re.sub(r'\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}', ' specipaddress ', comment)\n",
    "\n",
    "    # remove username\n",
    "    comment = re.sub(r\"\\[\\[User.*\\]\", ' specusername ', comment)\n",
    "    comment = re.sub(r\"\\[\\[User.*\\|\", ' specusername ', comment)\n",
    "    \n",
    "    # remove special characters\n",
    "    comment = re.sub(r'[^A-Za-z\\d\\' ]', '', comment)\n",
    "\n",
    "    # tokenization \n",
    "    tokens = comment.split()\n",
    "\n",
    "    # aphostophe replacement\n",
    "    tokens = [ appos[token] if token in appos else token for token in tokens]\n",
    "\n",
    "    # remove stopwords\n",
    "    tokens = [ token for token in tokens if not token in eng_stopwords ]\n",
    "\n",
    "    # stemming\n",
    "    tokens = [ lmtzr.lemmatize(token, 'v') for token in tokens]\n",
    "\n",
    "    return \" \".join(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "hint('Cleaning train set...')\n",
    "Xtr_text = train['comment_text'].apply(lambda c: preprocess(c))\n",
    "hint('Cleaning test set...')\n",
    "Xva_text = valid['comment_text'].apply(lambda c: preprocess(c))\n",
    "hint('Done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Making Sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing import text as ktxt, sequence\n",
    "\n",
    "vocab_max = 100000\n",
    "\n",
    "hint('Fitting the tokenizer...')\n",
    "tokenizer = ktxt.Tokenizer(num_words=vocab_max)\n",
    "tokenizer.fit_on_texts(Xtr_text)\n",
    "\n",
    "hint('Tokenizing...')\n",
    "Xtr = tokenizer.texts_to_sequences(Xtr_text)\n",
    "Xva = tokenizer.texts_to_sequences(Xva_text)\n",
    "\n",
    "hint('padding the sequences...')\n",
    "max_comment_length = 200  # padded/cropped comment length\n",
    "Xtr = sequence.pad_sequences(Xtr, maxlen=max_comment_length)\n",
    "Xva = sequence.pad_sequences(Xva, maxlen=max_comment_length)\n",
    "\n",
    "hint('Done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Prepare Embedding Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "emb_file = 'preembedding/glove.6B.300d.txt'\n",
    "\n",
    "hint(\"Loading pre-embedding file...\")\n",
    "emb = pd.read_table(emb_file, \" \", index_col=0, header=None, quoting=csv.QUOTE_NONE)\n",
    "\n",
    "hint(\"Preparing embedding matrix...\")\n",
    "embedding_dimension = 300\n",
    "embedding_matrix = np.random.normal(\n",
    "    emb.mean(axis=0), \n",
    "    emb.std(axis=0), \n",
    "    (vocab_max, embedding_dimension)\n",
    ")\n",
    "\n",
    "hint(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "hint(\"Constructing embedding matrix\")\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    if i < vocab_max and word in emb.index:\n",
    "        embedding_matrix[i] = emb.loc[word].as_matrix()\n",
    "\n",
    "hint(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "del emb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Model and Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Models ready\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Input, Dense, Embedding, CuDNNGRU, CuDNNLSTM, BatchNormalization\n",
    "from keras.layers import SpatialDropout1D, Dropout, TimeDistributed, Bidirectional\n",
    "from keras.layers import GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate\n",
    "from keras.layers.convolutional import Conv1D, MaxPooling1D\n",
    "from keras.optimizers import Adam\n",
    "from attention_decoder import AttentionDecoder\n",
    "\n",
    "def get_model(input_dim, embedding_matrix, lr):\n",
    "    sequence_input = Input(shape=(input_dim, ))\n",
    "    x = Embedding(vocab_max, embedding_dimension, weights=[embedding_matrix])(sequence_input)\n",
    "    x = SpatialDropout1D(0.5)(x)\n",
    "    x = Conv1D(filters=256, kernel_size=3, padding='same', activation='relu')(x)\n",
    "    x = MaxPooling1D(pool_size=2)(x)\n",
    "    x = SpatialDropout1D(0.2)(x)\n",
    "    x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n",
    "    avg_pool = GlobalAveragePooling1D()(x)\n",
    "    max_pool = GlobalMaxPooling1D()(x)\n",
    "    x = concatenate([avg_pool, max_pool])\n",
    "    x = Dropout(0.2)(x)\n",
    "    preds = Dense(1, activation=\"sigmoid\")(x)\n",
    "    model = Model(sequence_input, preds)\n",
    "    model.compile(\n",
    "        loss='binary_crossentropy',\n",
    "        optimizer=Adam(lr=lr), \n",
    "        metrics=['accuracy']\n",
    "    )\n",
    "    return model\n",
    "\n",
    "models = [ get_model(max_comment_length, embedding_matrix, 0.0004) for _ in range(6) ]\n",
    "print(\"Models ready\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class weight matrix: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.55344276, 5.17790217]),\n",
       " array([ 0.50518818, 48.68649886]),\n",
       " array([0.52807585, 9.40444968]),\n",
       " array([  0.5015204 , 164.93023256]),\n",
       " array([ 0.52620384, 10.04058518]),\n",
       " array([ 0.50441366, 57.14234557])]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.utils import class_weight\n",
    "\n",
    "class_weights = []\n",
    "for i in range(6):\n",
    "    class_weights.append(class_weight.compute_class_weight(\n",
    "        'balanced', \n",
    "        np.unique(Ytr[:, i]), \n",
    "        Ytr[:, i]\n",
    "    ))\n",
    "print(\"Class weight matrix: \")\n",
    "class_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting class \"toxic\"\n",
      "Train on 127656 samples, validate on 31915 samples\n",
      "Epoch 1/10\n",
      "127656/127656 [==============================] - 69s 538us/step - loss: 0.1488 - acc: 0.9467 - val_loss: 0.1057 - val_acc: 0.9595\n",
      "Epoch 2/10\n",
      "111680/127656 [=========================>....] - ETA: 7s - loss: 0.1086 - acc: 0.9594"
     ]
    }
   ],
   "source": [
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "epochs = 10\n",
    "batch_size = 64\n",
    "val_loss_estop = EarlyStopping(monitor='val_loss')\n",
    "loss_estop = EarlyStopping(monitor='loss', patience=1)\n",
    "callback_list = [loss_estop, val_loss_estop]\n",
    "\n",
    "history = []\n",
    "for i, label in enumerate(labels):\n",
    "    print('Fitting class \\\"%s\\\"' % label)\n",
    "    history.append(models[i].fit(\n",
    "        Xtr, Ytr[:, i], \n",
    "        epochs=epochs, \n",
    "        batch_size=batch_size,\n",
    "        validation_data=(Xva, Yva[:, i]),\n",
    "        callbacks=callback_list,\n",
    "        class_weight=class_weights[i]\n",
    "    ))\n",
    "    print(' ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3 Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Yva_ = np.zeros((Yva.shape[0], len(labels)))\n",
    "for i, model in enumerate(models):\n",
    "    Yva_[:, i] = model.predict(Xva)[:, 0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "for i, label in enumerate(labels):\n",
    "    fpr, tpr, th = metrics.roc_curve(Yva[:, i], Yva_[:, i], pos_label=1)\n",
    "    print(\"AUC for label %s: %.4f\" % (label, metrics.auc(fpr, tpr)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}