Model_Pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_uuid": "f5d79359fe1530a043968435fee1e77279d4fa36"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "from random import choice\n",
    "import gc\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from contextlib import contextmanager\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "import lightgbm as lgb\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "from sklearn.metrics import f1_score # one more metric to evaluate\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "da4809a48f8e5b558399f1bd03e17ac790f3eda3"
   },
   "source": [
    "# Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_uuid": "0aac5499b6668b2e5331b330b223c3cc19c4ae0a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['df7_p_feav3.csv', 'df14_p_feav3.csv', 'df7_feav3.csv', 'df14_feav3.csv']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('../input/weekappv3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "28ed0d4e38566f52e7f977db2dbde10b54ff5af3"
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/predict-in-app-purchase/sample_submission_2.csv')\n",
    "df7 = pd.read_csv('../input/weekappv3/df7_feav3.csv')\n",
    "df14 = pd.read_csv('../input/weekappv3/df14_feav3.csv')\n",
    "df7_p = pd.read_csv('../input/weekappv3/df7_p_feav3.csv')\n",
    "df14_p = pd.read_csv('../input/weekappv3/df14_p_feav3.csv')\n",
    "submission = sub[['user_id_hash']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_uuid": "c78c684c8abd954b9b44a05423d9faddc3588d67"
   },
   "outputs": [],
   "source": [
    "# fillna with -1\n",
    "df7 = df7.fillna(-1)\n",
    "df14 = df14.fillna(-1)\n",
    "df7_p = df7_p.fillna(-1)\n",
    "df14_p = df14_p.fillna(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_uuid": "27bd8ac24bef39628438481e516734da0bfa0946"
   },
   "outputs": [],
   "source": [
    "def fea_importance(tree,fea):\n",
    "    importance_df = pd.DataFrame()\n",
    "    importance_df[\"feature\"] = fea\n",
    "    importance_df[\"importance\"] = tree.feature_importances_\n",
    "    return importance_df.sort_values(by='importance',ascending=False)\n",
    "    \n",
    "def threshold_search(y_true, y_proba):\n",
    "    '''\n",
    "    searching a threshold to find the best f1-score\n",
    "    '''\n",
    "    best_threshold = 0\n",
    "    best_score = 0\n",
    "    for threshold in [i * 0.01 for i in range(100)]:\n",
    "        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n",
    "        if score > best_score:\n",
    "            best_threshold = threshold\n",
    "            best_score = score\n",
    "#     search_result = {'threshold': best_threshold, 'f1': best_score} print if u want\n",
    "    return best_score\n",
    "\n",
    "def Model(model,trainX,trainy,nsplits=5):\n",
    "    nsplits=5\n",
    "    kf = StratifiedShuffleSplit(n_splits=nsplits,random_state=420)\n",
    "    df = pd.DataFrame(columns=['Average AUC', 'Average F_score'])\n",
    "\n",
    "    t = time.time()\n",
    "    all_auc = []\n",
    "    f_score = []\n",
    "    for train, test in kf.split(trainX, trainy):\n",
    "        X_train, X_test, y_train, y_test = trainX.loc[train], trainX.loc[test], trainy[train], trainy[test]\n",
    "        model.fit(X_train, y_train)\n",
    "        probabilities = model.predict_proba(X_test)\n",
    "        score = probabilities[:, 1]\n",
    "        preds = model.predict(X_test)\n",
    "        all_auc.append(roc_auc_score(y_test, score))\n",
    "        fscore = threshold_search(y_test, score)\n",
    "        f_score.append(fscore)\n",
    "    print(\"AUC and F1\",(np.mean(all_auc), np.mean(f_score)))\n",
    "    print(f\"Time use:{time.time()-t:.3f}s\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "82af134fccdf5ff17624fc88c7ed894e10663e8f"
   },
   "source": [
    "# 7-day Part\n",
    "\n",
    "two small parts\n",
    "- use all features\n",
    "- use top20 features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_uuid": "2c2e226fecdc8cbae00b0044ba7734169bd3d6c8"
   },
   "outputs": [],
   "source": [
    "drop_col7 = ['con_act_day_count_max_89', 'time_gap_min_9', 'time_gap_min_89',\n",
    "       'is96', 'con_act_max_89', 'con_act_day_count_min_9',\n",
    "       'con_act_day_count_total_9', 'next_pch_time_min_rec_mon',\n",
    "       'con_act_day_count_max_9', 'con_act_day_count_min_09',\n",
    "       'con_act_day_count_max_09', 'con_act_day_count_mean_9',\n",
    "       'con_act_day_count_total_09', 'con_act_max_9', 'time_gap_max_9',\n",
    "       'con_act_day_count_total_89', 'con_act_day_count_std_89',\n",
    "       'con_act_day_count_mean_89', 'is67', 'con_act_day_count_std_9',\n",
    "       'con_act_day_count_std_09','user_purchase_binary_14_days',\n",
    "             'user_purchase_binary_7_days','user_id_le','user_id_hash']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_uuid": "4cde9f5b123ffe42c06fca717a9e3d436192cbee"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC and F1 (0.9906990645273226, 0.5592126362889578)\n",
      "Time use:839.170s\n",
      "AUC and F1 (0.9897716516882262, 0.5372940751949701)\n",
      "Time use:662.639s\n"
     ]
    }
   ],
   "source": [
    "# Those params have been tuned already\n",
    "param1 = {'n_estimators': 100, 'learning_rate': 0.1,'reg_lambda': 20,'is_unbalance': True, 'boosting_type': 'gbdt', 'max_depth': 5, 'feature_fraction': 0.7, 'objective': 'binary', 'metric': 'auc'}\n",
    "param1_r = {'n_estimators': 200, 'class_weight': 'balanced', 'min_samples_split': 20, 'n_jobs': -1, 'min_samples_leaf': 4, 'criterion': 'entropy', 'max_features': 'auto', 'max_depth': 14}\n",
    "feats = [c for c in df7.columns if c not in drop_col7]\n",
    "X_7 = df7[feats]\n",
    "\n",
    "# X_7 = X_7.rename(columns={'7buy_mean_tr>.8_tr':\"7buy_mean_tr_more_80p_tr\",'7length<3_tr':\"7length_less_3_tr\"})\n",
    "X_7 = X_7.rename(columns={'7buy_mean_tr>.8_tr':\"7buy_mean_tr8_tr\",'7length<3_tr':\"7length3_tr\"}) # for xgb\n",
    "y_7 = df7['user_purchase_binary_7_days']\n",
    "\n",
    "LGB1 = lgb.LGBMClassifier(**param1)\n",
    "RF1 = RandomForestClassifier(**param1_r)\n",
    "LR1 = LogisticRegression(class_weight='balanced')\n",
    "XGB1 = XGBClassifier(eval_metric='auc')\n",
    "xgb1 = Model(XGB1,X_7,y_7,nsplits=5)\n",
    "rf1 = Model(RF1,X_7,y_7,nsplits=5)\n",
    "# lr1 = Model(LR1,X_7,y_7,nsplits=5) # really slow\n",
    "# lgb1 = Model(LGB1,X_7,y_7,nsplits=5) # perform bad\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_uuid": "c6af81d6b54086652df555acbb4fb080eb67e08c"
   },
   "outputs": [],
   "source": [
    "# np.array(fea_importance(lgb1,feats)['feature'][:20])\n",
    "# print('Plotting feature importances...')\n",
    "# ax = lgb.plot_importance(lgb1, max_num_features=40,figsize=(15,10))\n",
    "# plt.show()\n",
    "# np.array(list(set(fea_importance(rf1,feats)[-30:]['feature']).intersection(set(fea_importance(lgb1,feats)[-30:]['feature']))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "86307348ee8afb5702d8a2f78c72942b99d56564"
   },
   "source": [
    "# 14-day Part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_uuid": "20ac69abe3bb61826929c648884a1b195c75f60c"
   },
   "outputs": [],
   "source": [
    "drop_col14 = ['con_pch_day_count_std_rec_mon', 'con_act_day_count_min_78',\n",
    "       'con_act_day_count_total_78', 'con_pch_day_count_total_rec_mon',\n",
    "       'time_gap_max_8', 'time_gap_min_8', 'is96',\n",
    "       'next_pch_time_min_rec_mon', 'con_act_day_count_mean_8',\n",
    "       'con_act_day_count_max_8', 'con_act_max_8', 'is67',\n",
    "       'con_act_day_count_mean_78', 'con_act_day_count_std_8',\n",
    "       'con_act_day_count_min_08', 'user_eve_day_count_78',\n",
    "       'con_act_day_count_min_8', 'time_gap_std_8',\n",
    "       'con_act_day_count_std_78', 'time_gap_mean_8','user_purchase_binary_14_days',\n",
    "        'user_purchase_binary_7_days','user_id_le','user_id_hash']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_uuid": "0cf80c7c30a4e532869caa0ee1a4a168f9af15cd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC and F1 (0.9930259154268327, 0.6479112807352301)\n",
      "Time use:839.585s\n",
      "AUC and F1 (0.9936688345454693, 0.6525635744578292)\n",
      "Time use:321.664s\n",
      "AUC and F1 (0.9916809216265332, 0.5991233159946361)\n",
      "Time use:537.967s\n"
     ]
    }
   ],
   "source": [
    "# param2 = {'n_estimators': 100, 'learning_rate': 0.1,'reg_lambda': 20,'is_unbalance': True, 'boosting_type': 'gbdt', 'max_depth': 7, 'feature_fraction': 0.7, 'objective': 'binary', 'metric': 'auc'}\n",
    "\n",
    "# Those params have been tuned already\n",
    "param2 = {'n_estimators': 150, 'bagging_fraction': 0.9, 'learning_rate': 0.1, 'is_unbalance': True, 'max_bin': 25, 'boosting_type': 'dart', 'max_depth': 5, 'feature_fraction': 0.8, 'lambda_l1': 40, 'objective': 'binary', 'metric': 'auc'}\n",
    "param2_r = {'n_estimators': 140, 'max_features': 'auto', 'n_jobs': -1, 'max_depth': 15, 'min_samples_leaf': 4, 'class_weight': 'balanced', 'min_samples_split': 8}\n",
    "\n",
    "feats2 = [c for c in df14.columns if c not in drop_col14]\n",
    "X_14 = df14[feats2]\n",
    "X_14 = X_14.rename(columns={'14buy_mean_tr>.8_tr':\"14buy_mean_tr8_tr\",'14length<3_tr':\"14length3_tr\"})\n",
    "y_14 = df14['user_purchase_binary_14_days']\n",
    "\n",
    "LGB2 = lgb.LGBMClassifier(**param2)\n",
    "RF2 = RandomForestClassifier(**param2_r)\n",
    "LR2 = LogisticRegression(class_weight='balanced')\n",
    "XGB2 = XGBClassifier(eval_metric='auc')\n",
    "xgb2 = Model(XGB2,X_14,y_14,nsplits=5)\n",
    "lgb2 = Model(LGB2,X_14,y_14,nsplits=5)\n",
    "rf2 = Model(RF2,X_14,y_14,nsplits=5)\n",
    "# lr2 = Model(LR2,X_14,y_14,nsplits=5)\n",
    "# fea_importance(rf2,rf_f20_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "_uuid": "87e01399c6eb9ea4799961042e9098ede3c32b53"
   },
   "outputs": [],
   "source": [
    "## use this to get top20\n",
    "# fea_importance(lgb2,feats2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_uuid": "283d3eb3600a9c3eb7e58a8df541dd7fdc419b3a"
   },
   "outputs": [],
   "source": [
    "# print('Plotting feature importances...')\n",
    "# ax = lgb.plot_importance(lgb2, max_num_features=40,figsize=(15,10))\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "855d2d8333c39bdc3ac4dd5611977bcacb77963a"
   },
   "source": [
    "# prediction\n",
    "- 7-day part\n",
    "- 14-day part"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "706c5e059b3e8f6872b33afec2d0be7e9de68c0c"
   },
   "source": [
    "# 7-day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_uuid": "aeee62a2c4a0e2d71e1f1c7bcae48b37cd6889ca"
   },
   "outputs": [],
   "source": [
    "drop_col7_p = ['con_act_day_count_max_910', 'time_gap_min_10', 'time_gap_min_910',\n",
    "       'is96', 'con_act_max_910', 'con_act_day_count_min_10',\n",
    "       'con_act_day_count_total_10', 'next_pch_time_min_rec_mon',\n",
    "       'con_act_day_count_max_10', 'con_act_day_count_min_010',\n",
    "       'con_act_day_count_max_010', 'con_act_day_count_mean_10',\n",
    "       'con_act_day_count_total_010', 'con_act_max_10', 'time_gap_max_10',\n",
    "       'con_act_day_count_total_910', 'con_act_day_count_std_910',\n",
    "       'con_act_day_count_mean_910', 'is67', 'con_act_day_count_std_10',\n",
    "       'con_act_day_count_std_010','user_purchase_binary_14_days',\n",
    "        'user_purchase_binary_7_days','user_id_le','user_id_hash']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_uuid": "4341dbb23d3ae0c75161c7df2bcece331c95cc72"
   },
   "outputs": [],
   "source": [
    "# use all feats1\n",
    "X_7_p = df7_p[[c for c in df7_p.columns if c not in drop_col7_p]]\n",
    "X_7_p = X_7.rename(columns={'7buy_mean>.8_p':'7buy_mean8_p','7length<3_p':'7length3_p'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_uuid": "47a67d600d99f8989e5643a9eb889fa759684291"
   },
   "outputs": [],
   "source": [
    "# use all feats(just rf+lr) lgb is so bad!\n",
    "pred_rf7 = rf1.predict_proba(X_7_p)[:, 1]\n",
    "pred_xgb7 = xgb1.predict_proba(X_7_p)[:, 1]\n",
    "# pred_lr7 = lr1.predict_proba(X_7_p)[:, 1]\n",
    "\n",
    "# pred =  1/3*pred_rf7+1/3*pred_xgb7+1/3*pred_lr7\n",
    "pred = 1/2*pred_xgb7+1/2*pred_rf7\n",
    "df7_p['user_purchase_binary_7_days'] = pred\n",
    "pred_df = df7_p[['user_id_hash','user_purchase_binary_7_days']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "2653fc5cab83c220c66078d2213c3340ee756ad1"
   },
   "source": [
    "# 14-day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_uuid": "dcedc7c54971d2fd7e74ecbb0b5401fcd94d1965"
   },
   "outputs": [],
   "source": [
    "drop_col14_p = ['con_pch_day_count_std_rec_mon', 'con_act_day_count_min_910',\n",
    "       'con_act_day_count_total_910', 'con_pch_day_count_total_rec_mon',\n",
    "       'time_gap_max_10', 'time_gap_min_10', 'is96',\n",
    "       'next_pch_time_min_rec_mon', 'con_act_day_count_mean_10',\n",
    "       'con_act_day_count_max_10', 'con_act_max_10', 'is67',\n",
    "       'con_act_day_count_mean_910', 'con_act_day_count_std_10',\n",
    "       'con_act_day_count_min_010', 'user_eve_day_count_910',\n",
    "       'con_act_day_count_min_10', 'time_gap_std_10',\n",
    "       'con_act_day_count_std_910', 'time_gap_mean_10',\n",
    "        'user_purchase_binary_14_days','user_purchase_binary_7_days',\n",
    "         'user_id_le','user_id_hash']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "_uuid": "13e65288a46b6441459f3fc1af8d1052ee5b2c7a"
   },
   "outputs": [],
   "source": [
    "# use all feats\n",
    "X_14_p = df14_p[[c for c in df14_p.columns if c not in drop_col14_p]]\n",
    "X_14_p = X_14.rename(columns={'14buy_mean>.8_p':'14buy_mean8_p','14length<3_p':'14length3_p'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "_uuid": "d6ad77fd2fa0afd031afd8f54a20fb3269120e8e"
   },
   "outputs": [],
   "source": [
    "pred_lg14 = lgb2.predict_proba(X_14_p)[:, 1]\n",
    "pred_rf14 = rf2.predict_proba(X_14_p)[:, 1]\n",
    "pred_xgb14 = xgb2.predict_proba(X_14_p)[:, 1]\n",
    "pred2 =  1/3*pred_lg14+1/3*pred_rf14+1/3*pred_xgb14\n",
    "df14_p['user_purchase_binary_14_days'] = pred2\n",
    "pred_df2 = df14_p[['user_id_hash','user_purchase_binary_14_days']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "_uuid": "629d78f1bdbbc94c0cf2787bb6cf697b3925929e"
   },
   "outputs": [],
   "source": [
    "submission = pd.merge(submission,pred_df,on=['user_id_hash'],how='left')\n",
    "submission = pd.merge(submission,pred_df2,on=['user_id_hash'],how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "_uuid": "223e88d31a951982434e2a052ccbba5a4f6dce7c"
   },
   "outputs": [],
   "source": [
    "def impute_mean(df):\n",
    "    for c in df.columns:\n",
    "        if c not in ['user_id_hash']:\n",
    "            df[c] = df[c].fillna(0)\n",
    "    return df\n",
    "\n",
    "submission = impute_mean(submission)\n",
    "submission.to_csv('submission.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}