train_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Train Models\n",
    "<div style=\"color:red; font-size:14px;\">!! Don't define functions here, import them from utils.py</div>\n",
    "\n",
    "This notebook contains the code needed to train and store models to disk.\n",
    "\n",
    "Remember that if you use a function with a random state you have to fix it to a number so that the results are reproducible."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Imports"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sklearn\n",
    "from sklearn import *\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "from utils import *"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "home_dir = os.environ['HOME']\n",
    "path_folder_quora = home_dir + '/Datasets/QuoraQuestionPairs'"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "path_folder_quora"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(os.path.join(path_folder_quora, 'quora_train_data.csv'))\n",
    "test_df = pd.read_csv(os.path.join(path_folder_quora, 'quora_test_data.csv'))\n",
    "\n",
    "A_df, te_df = sklearn.model_selection.train_test_split(train_df,\n",
    "                                                       test_size=0.05,\n",
    "                                                       random_state=123)\n",
    "tr_df, va_df = sklearn.model_selection.train_test_split(A_df,\n",
    "                                                        test_size=0.05,\n",
    "                                                        random_state=123)\n",
    "y_tr = tr_df['is_duplicate'].values\n",
    "X_tr_df = tr_df.drop(['is_duplicate'], axis =1)\n",
    "\n",
    "y_va = va_df['is_duplicate'].values\n",
    "X_va_df = va_df.drop(['is_duplicate'], axis =1)\n",
    "\n",
    "y_te = te_df['is_duplicate'].values\n",
    "X_te_df = te_df.drop(['is_duplicate'], axis =1)\n",
    "\n",
    "print('X_tr_df.shape=',X_tr_df.shape)\n",
    "print('y_tr.shape=',y_tr.shape)\n",
    "print('X_va.shape=',X_va_df.shape)\n",
    "print('y_va_df.shape=',y_tr.shape)\n",
    "print('X_te.shape=',X_te_df.shape)\n",
    "print('y_tr_df.shape=',y_tr.shape)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Explore data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "train_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Create question database"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# concatenate qid1 and qid2 into a new column called \"qid\"\n",
    "qid1 = train_df[['qid1', 'question1']].rename(columns={'qid1': 'qid', 'question1': 'question'})\n",
    "qid2 = train_df[['qid2', 'question2']].rename(columns={'qid2': 'qid', 'question2': 'question'})\n",
    "qid_df = pd.concat([qid1, qid2])\n",
    "\n",
    "# drop any duplicate rows based on \"qid\" column\n",
    "qid_df = qid_df.drop_duplicates(subset=['qid'])\n",
    "\n",
    "# sort the dataframe by \"qid\"\n",
    "qid_df = qid_df.sort_values(by=['qid'])\n",
    "\n",
    "# reset the index of the dataframe\n",
    "qid_df = qid_df.reset_index(drop=True)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Simple Solution"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Build corpus combining all questions in a list\n",
    "all_q1 = list(X_tr_df[\"question1\"])\n",
    "all_q2 = list(X_tr_df[\"question2\"])\n",
    "all_questions = all_q1 + all_q2\n",
    "\n",
    "len(all_questions)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Cast lists as strings\n",
    "all_questions = cast_list_as_strings(all_questions)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Train and transform using Count Vectorizer\n",
    "count_vectorizer = sklearn.feature_extraction.text.CountVectorizer(ngram_range=(1,1))\n",
    "count_vectorizer.fit(all_questions)\n",
    "\n",
    "X_tr_q1q2 = get_features_from_df(X_tr_df, count_vectorizer)\n",
    "X_va_q1q2 = get_features_from_df(X_va_df, count_vectorizer)\n",
    "X_te_q1q2  = get_features_from_df(X_te_df, count_vectorizer)\n",
    "\n",
    "X_tr_q1q2.shape, tr_df.shape, X_va_q1q2.shape, va_df.shape, te_df.shape, X_te_q1q2.shape"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Train model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Train Logistic Regression Model\n",
    "lr_model = sklearn.linear_model.LogisticRegression(solver=\"liblinear\",\n",
    "                                                   random_state=123)\n",
    "lr_model.fit(X_tr_q1q2, y_tr)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Save model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "if not os.path.isdir(\"model_artifacts\"):\n",
    "    os.mkdir(\"model_artifacts\")\n",
    "\n",
    "if not os.path.isdir(\"model_artifacts/simple_solution\"):\n",
    "        os.mkdir(\"model_artifacts/simple_solution\")\n",
    "        # Save model and validation and test datasets\n",
    "        with open('model_artifacts/simple_solution/lr_model.pkl', 'wb') as file:\n",
    "            pickle.dump(lr_model, file)\n",
    "        with open('model_artifacts/simple_solution/X_tr_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_tr_q1q2, file)\n",
    "        with open('model_artifacts/simple_solution/y_tr.pkl', 'wb') as file:\n",
    "            pickle.dump(y_tr, file)\n",
    "        with open('model_artifacts/simple_solution/X_va_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_va_q1q2, file)\n",
    "        with open('model_artifacts/simple_solution/y_va.pkl', 'wb') as file:\n",
    "            pickle.dump(y_va, file)\n",
    "        with open('model_artifacts/simple_solution/X_te_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_te_q1q2, file)\n",
    "        with open('model_artifacts/simple_solution/y_te.pkl', 'wb') as file:\n",
    "            pickle.dump(y_te, file)\n",
    "        with open('model_artifacts/simple_solution/qid_df.pkl', 'wb') as file:\n",
    "            pickle.dump(qid_df, file)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Improvement proposals"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Create copy of question database to apply preprocessing for improve baseline solution\n",
    "qid_df_preprocess = qid_df.copy()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "<div class=\"alert\" style=\"padding: 20px;background-color: #2cbc84; color: white; margin-bottom: 15px;\">\n",
    "Baseline solution\n",
    "</div>"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Text preprocessing\n",
    "- Contractions and abbreviations: normalize_text\n",
    "- Remove punctuation: remove_punctuation\n",
    "- Spellchecker\n",
    "- Remove stopwords: remove_stopwords\n",
    "- Remove accents: remove_accents\n",
    "- Special tokens: special_tokens\n",
    "- Normalize spaces: normalize_spaces"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Cast list as strings"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df_preprocess['question'] = cast_list_as_strings(list(qid_df_preprocess[\"question\"]))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df_preprocess.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Text Normalization"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Dictionary of common contractions and their expanded form\n",
    "contractions_dict = {\n",
    "    \"ain't\": \"are not\",\n",
    "    \"can't\": \"cannot\",\n",
    "    \"could've\": \"could have\",\n",
    "    \"couldn't\": \"could not\",\n",
    "    \"didn't\": \"did not\",\n",
    "    \"doesn't\": \"does not\",\n",
    "    \"don't\": \"do not\",\n",
    "    \"hadn't\": \"had not\",\n",
    "    \"hasn't\": \"has not\",\n",
    "    \"haven't\": \"have not\",\n",
    "    \"he'd\": \"he would\",\n",
    "    \"he'll\": \"he will\",\n",
    "    \"he's\": \"he is\",\n",
    "    \"I'd\": \"I would\",\n",
    "    \"I'll\": \"I will\",\n",
    "    \"I'm\": \"I am\",\n",
    "    \"I've\": \"I have\",\n",
    "    \"isn't\": \"is not\",\n",
    "    \"it's\": \"it is\",\n",
    "    \"let's\": \"let us\",\n",
    "    \"might've\": \"might have\",\n",
    "    \"must've\": \"must have\",\n",
    "    \"shan't\": \"shall not\",\n",
    "    \"she'd\": \"she would\",\n",
    "    \"she'll\": \"she will\",\n",
    "    \"she's\": \"she is\",\n",
    "    \"should've\": \"should have\",\n",
    "    \"shouldn't\": \"should not\",\n",
    "    \"that's\": \"that is\",\n",
    "    \"there's\": \"there is\",\n",
    "    \"they'd\": \"they would\",\n",
    "    \"they'll\": \"they will\",\n",
    "    \"they're\": \"they are\",\n",
    "    \"they've\": \"they have\",\n",
    "    \"wasn't\": \"was not\",\n",
    "    \"we'd\": \"we would\",\n",
    "    \"we'll\": \"we will\",\n",
    "    \"we're\": \"we are\",\n",
    "    \"we've\": \"we have\",\n",
    "    \"weren't\": \"were not\",\n",
    "    \"what'll\": \"what will\",\n",
    "    \"what're\": \"what are\",\n",
    "    \"what's\": \"what is\",\n",
    "    \"what've\": \"what have\",\n",
    "    \"where's\": \"where is\",\n",
    "    \"who'd\": \"who would\",\n",
    "    \"who'll\": \"who will\",\n",
    "    \"who're\": \"who are\",\n",
    "    \"who's\": \"who is\",\n",
    "    \"who've\": \"who have\",\n",
    "    \"won't\": \"will not\",\n",
    "    \"would've\": \"would have\",\n",
    "    \"wouldn't\": \"would not\",\n",
    "    \"you'd\": \"you would\",\n",
    "    \"you'll\": \"you will\",\n",
    "    \"you're\": \"you are\",\n",
    "    \"you've\": \"you have\"\n",
    "}\n",
    "\n",
    "# Dictionary of common abbreviations and their full form\n",
    "abbreviations_dict = {\n",
    "    \"aka\": \"also known as\",\n",
    "    \"asap\": \"as soon as possible\",\n",
    "    \"btw\": \"by the way\",\n",
    "    \"etc\": \"et cetera\",\n",
    "    \"e.g.\": \"for example\",\n",
    "    \"i.e.\": \"that is\",\n",
    "    \"lol\": \"laugh out loud\",\n",
    "    \"omg\": \"oh my god\",\n",
    "    \"thx\": \"thanks\",\n",
    "    \"wtf\": \"what the fuck\"\n",
    "}\n",
    "\n",
    "qid_df_preprocess['question'] = qid_df_preprocess['question'].apply(lambda x: normalize_text(x, contractions_dict, abbreviations_dict))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df_preprocess.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Remove punctuation"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df_preprocess['question'] = qid_df_preprocess['question'].apply(lambda x: remove_punctuation(x))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df_preprocess.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Remove stopwords"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "stop_words = set([\n",
    "    'the', 'and', 'to', 'in', 'of', 'that', 'is', 'it', 'for',\n",
    "    'on', 'this', 'you', 'be', 'are', 'or', 'from', 'at', 'by', 'we',\n",
    "    'an', 'not', 'have', 'has', 'but', 'as', 'if', 'so', 'they', 'their',\n",
    "    'was', 'were','some', 'there', 'these', 'those', 'than', 'then', 'been', 'also',\n",
    "    'much', 'many', 'other'\n",
    "])\n",
    "\n",
    "qid_df_preprocess['question'] = qid_df_preprocess['question'].apply(lambda x: remove_stopwords(x, stop_words))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df_preprocess.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Remove accents"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df_preprocess['question'] = qid_df_preprocess['question'].apply(lambda x: remove_accents(x))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df_preprocess.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Special tokens"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from collections import Counter, defaultdict\n",
    "word_counts = Counter(word for sentence in all_questions for word in tokenize_text(sentence))\n",
    "# Create a defaultdict\n",
    "word_counts = defaultdict(lambda: 0, word_counts)\n",
    "# Words that only appears one\n",
    "word_counts_one = {k: v for k, v in word_counts.items() if v == 1}\n",
    "\n",
    "qid_df_preprocess['question'] = qid_df_preprocess['question'].apply(lambda x: special_tokens(x, word_counts_one))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df_preprocess.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Normalize spaces"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df_preprocess['question'] = qid_df_preprocess['question'].apply(lambda x: normalize_spaces(x))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "qid_df_preprocess.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Save preprocessed question"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "if not os.path.isdir(\"model_artifacts\"):\n",
    "    os.mkdir(\"model_artifacts\")\n",
    "with open('model_artifacts/qid_df.pkl', 'wb') as file:\n",
    "        pickle.dump(qid_df_preprocess, file)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Feature engineering"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "We start by replacing the questions in our training, validation and test sets by the preprocessed questions."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# ----------------------- TRAINING SET ___________________\n",
    "X_tr_df = X_tr_df.merge(qid_df_preprocess, left_on='qid1', right_on='qid', how='left').drop(columns=['qid'])\n",
    "X_tr_df = X_tr_df.merge(qid_df_preprocess, left_on='qid2', right_on='qid', how='left', suffixes=['_1', '_2']).drop(columns=['qid'])\n",
    "\n",
    "# replace the values in the question1 and question2 columns\n",
    "X_tr_df['question1'] = X_tr_df['question_1']\n",
    "X_tr_df['question2'] = X_tr_df['question_2']\n",
    "\n",
    "# drop the additional question_1 and question_2 columns\n",
    "X_tr_df = X_tr_df.drop(columns=['question_1', 'question_2'])\n",
    "\n",
    "# remove empty questions\n",
    "mask = (X_tr_df['question1'].str.len() > 0) & (X_tr_df['question2'].str.len() > 0)\n",
    "X_tr_df = X_tr_df[mask]\n",
    "y_tr = y_tr[mask]\n",
    "\n",
    "# ----------------------- VALIDATION SET ___________________\n",
    "X_va_df = X_va_df.merge(qid_df_preprocess, left_on='qid1', right_on='qid', how='left').drop(columns=['qid'])\n",
    "X_va_df = X_va_df.merge(qid_df_preprocess, left_on='qid2', right_on='qid', how='left', suffixes=['_1', '_2']).drop(columns=['qid'])\n",
    "\n",
    "# replace the values in the question1 and question2 columns\n",
    "X_va_df['question1'] = X_va_df['question_1']\n",
    "X_va_df['question2'] = X_va_df['question_2']\n",
    "\n",
    "# drop the additional question_1 and question_2 columns\n",
    "X_va_df = X_va_df.drop(columns=['question_1', 'question_2'])\n",
    "\n",
    "# remove empty questions\n",
    "mask = (X_va_df['question1'].str.len() > 0) & (X_va_df['question2'].str.len() > 0)\n",
    "X_va_df = X_va_df[mask]\n",
    "y_va = y_va[mask]\n",
    "\n",
    "# ----------------------- TEST SET ___________________\n",
    "X_te_df = X_te_df.merge(qid_df_preprocess, left_on='qid1', right_on='qid', how='left').drop(columns=['qid'])\n",
    "X_te_df = X_te_df.merge(qid_df_preprocess, left_on='qid2', right_on='qid', how='left', suffixes=['_1', '_2']).drop(columns=['qid'])\n",
    "\n",
    "# replace the values in the question1 and question2 columns\n",
    "X_te_df['question1'] = X_te_df['question_1']\n",
    "X_te_df['question2'] = X_te_df['question_2']\n",
    "\n",
    "# drop the additional question_1 and question_2 columns\n",
    "X_te_df = X_te_df.drop(columns=['question_1', 'question_2'])\n",
    "\n",
    "# remove empty questions\n",
    "mask = (X_te_df['question1'].str.len() > 0) & (X_te_df['question2'].str.len() > 0)\n",
    "X_te_df = X_te_df[mask]\n",
    "y_te = y_te[mask]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Features"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['unique_words_count'] = X_tr_df.apply(lambda x: num_of_unique_words(x['question1'], x['question2']), axis=1)\n",
    "X_va_df['unique_words_count'] = X_va_df.apply(lambda x: num_of_unique_words(x['question1'], x['question2']), axis=1)\n",
    "X_te_df['unique_words_count'] = X_te_df.apply(lambda x: num_of_unique_words(x['question1'], x['question2']), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['diff_word_count'] = X_tr_df.apply(lambda x: difference_word_count(x['question1'], x['question2']), axis=1)\n",
    "X_va_df['diff_word_count'] = X_va_df.apply(lambda x: difference_word_count(x['question1'], x['question2']), axis=1)\n",
    "X_te_df['diff_word_count'] = X_te_df.apply(lambda x: difference_word_count(x['question1'], x['question2']), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['common_word_count'] = X_tr_df.apply(lambda x: common_words_count(x), axis=1)\n",
    "X_va_df['common_word_count'] = X_va_df.apply(lambda x: common_words_count(x), axis=1)\n",
    "X_te_df['common_word_count'] = X_te_df.apply(lambda x: common_words_count(x), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['common_word_ratio'] = X_tr_df.apply(lambda x: common_words_ratio(x), axis=1)\n",
    "X_va_df['common_word_ratio'] = X_va_df.apply(lambda x: common_words_ratio(x), axis=1)\n",
    "X_te_df['common_word_ratio'] = X_te_df.apply(lambda x: common_words_ratio(x), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['first_word_same'] = X_tr_df.apply(lambda x: first_word_equal(x), axis=1)\n",
    "X_va_df['first_word_same'] = X_va_df.apply(lambda x: first_word_equal(x), axis=1)\n",
    "X_te_df['first_word_same'] = X_te_df.apply(lambda x: first_word_equal(x), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['last_word_same'] = X_tr_df.apply(lambda x: last_word_equal(x), axis=1)\n",
    "X_va_df['last_word_same'] = X_va_df.apply(lambda x: last_word_equal(x), axis=1)\n",
    "X_te_df['last_word_same'] = X_te_df.apply(lambda x: last_word_equal(x), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['fuzz_ratio'] = X_tr_df.apply(lambda x: fuzz_ratio(x), axis=1)\n",
    "X_va_df['fuzz_ratio'] = X_va_df.apply(lambda x: fuzz_ratio(x), axis=1)\n",
    "X_te_df['fuzz_ratio'] = X_te_df.apply(lambda x: fuzz_ratio(x), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['diff_char_count'] = X_tr_df.apply(lambda x: abs(num_of_characters(x['question1']) - num_of_characters(x['question2'])), axis=1)\n",
    "X_va_df['diff_char_count'] = X_va_df.apply(lambda x: abs(num_of_characters(x['question1']) - num_of_characters(x['question2'])), axis=1)\n",
    "X_te_df['diff_char_count'] = X_te_df.apply(lambda x: abs(num_of_characters(x['question1']) - num_of_characters(x['question2'])), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['unique_words_ratio'] = X_tr_df.apply(lambda x: total_unique_words_ratio(x['question1'], x['question2']), axis=1)\n",
    "X_va_df['unique_words_ratio'] = X_va_df.apply(lambda x: total_unique_words_ratio(x['question1'], x['question2']), axis=1)\n",
    "X_te_df['unique_words_ratio'] = X_te_df.apply(lambda x: total_unique_words_ratio(x['question1'], x['question2']), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['rare_word_count'] = X_tr_df.apply(lambda x: abs(rare_word_count(x['question1'], word_counts, 5) - rare_word_count(x['question2'], word_counts, 5)), axis=1)\n",
    "X_va_df['rare_word_count'] = X_va_df.apply(lambda x: abs(rare_word_count(x['question1'], word_counts, 5) - rare_word_count(x['question2'], word_counts, 5)), axis=1)\n",
    "X_te_df['rare_word_count'] = X_te_df.apply(lambda x: abs(rare_word_count(x['question1'], word_counts, 5) - rare_word_count(x['question2'], word_counts, 5)), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['diff_syllable_count'] = X_tr_df.apply(lambda x: abs(count_sentence_syllables(x['question1']) - count_sentence_syllables(x['question2'])), axis=1)\n",
    "X_va_df['diff_syllable_count'] = X_va_df.apply(lambda x: abs(count_sentence_syllables(x['question1']) - count_sentence_syllables(x['question2'])), axis=1)\n",
    "X_te_df['diff_syllable_count'] = X_te_df.apply(lambda x: abs(count_sentence_syllables(x['question1']) - count_sentence_syllables(x['question2'])), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['diff_flesch_reading_easy'] = X_tr_df.apply(lambda x: abs(Flesch_Reading_Ease(x['question1']) - Flesch_Reading_Ease(x['question2'])), axis=1)\n",
    "X_va_df['diff_flesch_reading_easy'] = X_va_df.apply(lambda x: abs(Flesch_Reading_Ease(x['question1']) - Flesch_Reading_Ease(x['question2'])), axis=1)\n",
    "X_te_df['diff_flesch_reading_easy'] = X_te_df.apply(lambda x: abs(Flesch_Reading_Ease(x['question1']) - Flesch_Reading_Ease(x['question2'])), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df['diff_flesch_grade_level'] = X_tr_df.apply(lambda x: abs(Flesch_Grade_Level(x['question1']) - Flesch_Grade_Level(x['question2'])), axis=1)\n",
    "X_va_df['diff_flesch_grade_level'] = X_va_df.apply(lambda x: abs(Flesch_Grade_Level(x['question1']) - Flesch_Grade_Level(x['question2'])), axis=1)\n",
    "X_te_df['diff_flesch_grade_level'] = X_te_df.apply(lambda x: abs(Flesch_Grade_Level(x['question1']) - Flesch_Grade_Level(x['question2'])), axis=1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Create question embeddings"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import fasttext.util"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "fasttext.util.download_model('en', if_exists='ignore')  # English\n",
    "ft_model = fasttext.load_model('cc.en.300.bin')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "print('===== Computing fasttext embeddings for training set =====')\n",
    "X_tr_q1q2 = get_fasttext_embeddings_and_features(X_tr_df, ft_model)\n",
    "print('===== Computing fasttext embeddings for validation set =====')\n",
    "X_va_q1q2 = get_fasttext_embeddings_and_features(X_va_df, ft_model)\n",
    "print('===== Computing fasttext embeddings for test set =====')\n",
    "X_te_q1q2 = get_fasttext_embeddings_and_features(X_te_df, ft_model)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_q1q2_base = X_tr_q1q2.drop(['question1', 'question2'],axis = 1)\n",
    "X_va_q1q2_base = X_va_q1q2.drop(['question1', 'question2'],axis = 1)\n",
    "X_te_q1q2_base = X_te_q1q2.drop(['question1', 'question2'],axis = 1)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Train Model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import time\n",
    "import xgboost as xgb\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "xgb_model = xgb.XGBClassifier(random_state=123)\n",
    "xgb_model.fit(X_tr_q1q2_base.drop(['id','qid1','qid2'],axis = 1), y_tr)\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "print(\"Training time:\", end_time - start_time, \"seconds\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Save Model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "if not os.path.isdir(\"model_artifacts\"):\n",
    "    os.mkdir(\"model_artifacts\")\n",
    "\n",
    "if not os.path.isdir(\"model_artifacts/improved_solution_baseline\"):\n",
    "        os.mkdir(\"model_artifacts/improved_solution_baseline\")\n",
    "        # Save model and validation and test datasets\n",
    "        with open('model_artifacts/improved_solution_baseline/xgb_model.pkl', 'wb') as file:\n",
    "            pickle.dump(xgb_model, file)\n",
    "        with open('model_artifacts/improved_solution_baseline/X_tr_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_tr_q1q2_base, file)\n",
    "        with open('model_artifacts/improved_solution_baseline/y_tr.pkl', 'wb') as file:\n",
    "            pickle.dump(y_tr, file)\n",
    "        with open('model_artifacts/improved_solution_baseline/X_va_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_va_q1q2_base, file)\n",
    "        with open('model_artifacts/improved_solution_baseline/y_va.pkl', 'wb') as file:\n",
    "            pickle.dump(y_va, file)\n",
    "        with open('model_artifacts/improved_solution_baseline/X_te_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_te_q1q2_base, file)\n",
    "        with open('model_artifacts/improved_solution_baseline/y_te.pkl', 'wb') as file:\n",
    "            pickle.dump(y_te, file)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "<div class=\"alert\" style=\"padding: 10px;background-color: #6bd0a9; color: white; margin-bottom: 15px; font-size:17px\">\n",
    "Baseline w/ CountVectorizer features\n",
    "</div>"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Add count_vectorizer"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Build corpus combining all questions in a list\n",
    "all_q1 = list(X_tr_q1q2[\"question1\"])\n",
    "all_q2 = list(X_tr_q1q2[\"question2\"])\n",
    "all_questions = all_q1 + all_q2"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "count_vectorizer = sklearn.feature_extraction.text.CountVectorizer(ngram_range=(1,1))\n",
    "count_vectorizer.fit(all_questions)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_q1q2_cv = get_countvectorizer_features(X_tr_q1q2, count_vectorizer)\n",
    "X_va_q1q2_cv = get_countvectorizer_features(X_va_q1q2, count_vectorizer)\n",
    "X_te_q1q2_cv = get_countvectorizer_features(X_te_q1q2, count_vectorizer)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Train Model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import time\n",
    "import xgboost as xgb\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "xgb_model_cv = xgb.XGBClassifier(random_state=123)\n",
    "xgb_model_cv.fit(X_tr_q1q2_cv[:, 3:], y_tr)\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "print(\"Training time:\", end_time - start_time, \"seconds\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Save Model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "if not os.path.isdir(\"model_artifacts\"):\n",
    "    os.mkdir(\"model_artifacts\")\n",
    "\n",
    "if not os.path.isdir(\"model_artifacts/improved_solution_baseline_cv\"):\n",
    "        os.mkdir(\"model_artifacts/improved_solution_baseline_cv\")\n",
    "        # Save model and validation and test datasets\n",
    "        with open('model_artifacts/improved_solution_baseline_cv/xgb_model.pkl', 'wb') as file:\n",
    "            pickle.dump(xgb_model_cv, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_cv/X_tr_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_tr_q1q2_cv, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_cv/y_tr.pkl', 'wb') as file:\n",
    "            pickle.dump(y_tr, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_cv/X_va_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_va_q1q2_cv, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_cv/y_va.pkl', 'wb') as file:\n",
    "            pickle.dump(y_va, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_cv/X_te_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_te_q1q2_cv, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_cv/y_te.pkl', 'wb') as file:\n",
    "            pickle.dump(y_te, file)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "<div class=\"alert\" style=\"padding: 10px;background-color: #6bd0a9; color: white; margin-bottom: 15px; font-size:17px\">\n",
    "Baseline w/ TF-IDF features\n",
    "</div>"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Add TF-IDF features"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "!python cython_utils/setup.py build_ext --build-lib=./cython_utils"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from cython_utils import tf_idf"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Create sklearn tfidf vectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tf_idf = TfidfVectorizer()\n",
    "tf_idf.fit(qid_df_preprocess['question'])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "X_tr_q1q2_tf = get_tfidf_features(X_tr_q1q2, tf_idf)\n",
    "X_va_q1q2_tf = get_tfidf_features(X_va_q1q2, tf_idf)\n",
    "X_te_q1q2_tf = get_tfidf_features(X_te_q1q2, tf_idf)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Train Model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import time\n",
    "import xgboost as xgb\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "xgb_model_tf = xgb.XGBClassifier(random_state=123)\n",
    "xgb_model_tf.fit(X_tr_q1q2_tf[:, 3:], y_tr)\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "print(\"Training time:\", end_time - start_time, \"seconds\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Save Model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "if not os.path.isdir(\"model_artifacts\"):\n",
    "    os.mkdir(\"model_artifacts\")\n",
    "\n",
    "if not os.path.isdir(\"model_artifacts/improved_solution_baseline_tf\"):\n",
    "        os.mkdir(\"model_artifacts/improved_solution_baseline_tf\")\n",
    "        # Save model and validation and test datasets\n",
    "        with open('model_artifacts/improved_solution_baseline_tf/xgb_model.pkl', 'wb') as file:\n",
    "            pickle.dump(xgb_model_tf, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_tf/X_tr_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_tr_q1q2_tf, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_tf/y_tr.pkl', 'wb') as file:\n",
    "            pickle.dump(y_tr, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_tf/X_va_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_va_q1q2_tf, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_tf/y_va.pkl', 'wb') as file:\n",
    "            pickle.dump(y_va, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_tf/X_te_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_te_q1q2_tf, file)\n",
    "        with open('model_artifacts/improved_solution_baseline_tf/y_te.pkl', 'wb') as file:\n",
    "            pickle.dump(y_te, file)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "<div class=\"alert\" style=\"padding: 20px;background-color: #2cbc84; color: white; margin-bottom: 15px;\">\n",
    "Improved solution with feature selection\n",
    "</div>"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Find most important features"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Retrieve the feature importance scores\n",
    "importance_scores = xgb_model.get_booster().get_score(importance_type='weight')\n",
    "\n",
    "# Sort the feature importance scores in descending order\n",
    "sorted_scores = sorted(importance_scores.items(), key=lambda x: x[1], reverse=True)\n",
    "print(sorted_scores)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### New dataset only with the most important features"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Select the top n features\n",
    "n = 300  # set the number of top features you want to select\n",
    "top_features = [(x[0][0:]) for x in sorted_scores[:n]]\n",
    "print(top_features)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "top_features.append('qid1')\n",
    "top_features.append('qid2')\n",
    "top_features.append('id')\n",
    "# Select only the top features in the training data\n",
    "X_tr_q1q2_top = X_tr_q1q2[top_features]\n",
    "# Select only the top features in the validation data\n",
    "X_va_q1q2_top = X_va_q1q2[top_features]\n",
    "# Select only the top features in the test data\n",
    "X_te_q1q2_top = X_te_q1q2[top_features]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "<div class=\"alert\" style=\"padding: 10px;background-color: #6bd0a9; color: white; margin-bottom: 15px; font-size:17px\">\n",
    "w/ XGBoost classifier\n",
    "</div>"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import time\n",
    "import xgboost as xgb\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "xgb_model_improve = xgb.XGBClassifier(random_state=123)\n",
    "xgb_model_improve.fit(X_tr_q1q2_top.drop(['id','qid1','qid2'],axis = 1), y_tr)\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "print(\"Training time:\", end_time - start_time, \"seconds\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "if not os.path.isdir(\"model_artifacts\"):\n",
    "    os.mkdir(\"model_artifacts\")\n",
    "\n",
    "if not os.path.isdir(\"model_artifacts/improved_solution_topfeatures\"):\n",
    "        os.mkdir(\"model_artifacts/improved_solution_topfeatures\")\n",
    "        # Save model and validation and test datasets\n",
    "        with open('model_artifacts/improved_solution_topfeatures/xgb_model.pkl', 'wb') as file:\n",
    "            pickle.dump(xgb_model_improve, file)\n",
    "        with open('model_artifacts/improved_solution_topfeatures/X_tr_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_tr_q1q2_top, file)\n",
    "        with open('model_artifacts/improved_solution_topfeatures/y_tr.pkl', 'wb') as file:\n",
    "            pickle.dump(y_tr, file)\n",
    "        with open('model_artifacts/improved_solution_topfeatures/X_va_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_va_q1q2_top, file)\n",
    "        with open('model_artifacts/improved_solution_topfeatures/y_va.pkl', 'wb') as file:\n",
    "            pickle.dump(y_va, file)\n",
    "        with open('model_artifacts/improved_solution_topfeatures/X_te_q1q2.pkl', 'wb') as file:\n",
    "            pickle.dump(X_te_q1q2_top, file)\n",
    "        with open('model_artifacts/improved_solution_topfeatures/y_te.pkl', 'wb') as file:\n",
    "            pickle.dump(y_te, file)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "<div class=\"alert\" style=\"padding: 10px;background-color: #6bd0a9; color: white; margin-bottom: 15px; font-size:17px\">\n",
    "w/ Random Forest classifier\n",
    "</div>"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import time\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "rf_model = RandomForestClassifier(max_depth = 5, random_state=123)\n",
    "rf_model.fit(X_tr_q1q2_top.drop(['id','qid1','qid2'],axis = 1), y_tr)\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "print(\"Training time:\", end_time - start_time, \"seconds\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "with open('model_artifacts/improved_solution_topfeatures/rf_model.pkl', 'wb') as file:\n",
    "            pickle.dump(rf_model, file)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "<div class=\"alert\" style=\"padding: 10px;background-color: #6bd0a9; color: white; margin-bottom: 15px; font-size:17px\">\n",
    "w/ Histogram-Based Gradient Boosting classifier\n",
    "</div>"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import time\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "hbgd_model = HistGradientBoostingClassifier(max_depth = 20,max_iter = 500,random_state=123)\n",
    "hbgd_model.fit(X_tr_q1q2_top.drop(['id','qid1','qid2'],axis = 1), y_tr)\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "print(\"Training time:\", end_time - start_time, \"seconds\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "with open('model_artifacts/improved_solution_topfeatures/hbgd_model.pkl', 'wb') as file:\n",
    "            pickle.dump(hbgd_model, file)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "<div class=\"alert\" style=\"padding: 10px;background-color: #6bd0a9; color: white; margin-bottom: 15px; font-size:17px\">\n",
    "w/ Ensembling\n",
    "XGBoost + HistGradientBoostingClassifier\n",
    "</div>"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "clf1 = xgb.XGBClassifier(random_state=123)\n",
    "clf3 = HistGradientBoostingClassifier(max_depth = 20,max_iter = 500,random_state=123)\n",
    "\n",
    "eclf1 = VotingClassifier(estimators=[('xgb', clf1), ('hgbc', clf3)], voting='soft')\n",
    "eclf1 = eclf1.fit(X_tr_q1q2_top.drop(['id','qid1','qid2'],axis = 1), y_tr)\n",
    "\n",
    "end_time = time.time()\n",
    "print(\"Training time:\", end_time - start_time, \"seconds\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "with open('model_artifacts/improved_solution_topfeatures/eclf1.pkl', 'wb') as file:\n",
    "            pickle.dump(eclf1, file)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}