DL on cont data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#proc_df\" data-toc-modified-id=\"proc_df-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>proc_df</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(120000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 120 seconds\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 4\n",
    "%autosave 120"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from fastai.imports import *\n",
    "from fastai.torch_imports import *\n",
    "from fastai.dataset import *\n",
    "from fastai.learner import *\n",
    "from fastai.structured import *\n",
    "from fastai.column_data import *\n",
    "import feather\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai/lib/python3.6/site-packages/pyarrow/pandas_compat.py:752: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.\n",
      "  labels, = index.labels\n"
     ]
    }
   ],
   "source": [
    "df_raw = feather.read_dataframe('all_train_cont')\n",
    "# df_test = feather.read_dataframe('all_test_cont')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dep = 'target'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for df in [df_raw]:\n",
    "    df.drop('price', axis=1, inplace=True)\n",
    "    for f in ['purchase_date']:\n",
    "        df[f] = df[f].astype(np.int64) * 1e-9\n",
    "    for c in ['avg_purchases_lag3', 'avg_purchases_lag6', 'avg_purchases_lag12', \n",
    "              'purchase_amount_sum_ratio', 'installments_ratio',\n",
    "              'month_diff_ratio',\n",
    "              'purchase_amount_sum_ratio']:\n",
    "        df[c] = df[c].replace(np.inf, df[c].median())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_cards = df_raw['card_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(201917,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_cards.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_cards = np.random.choice(train_cards, 40000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40000,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_cards.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idx = df_raw.index[df_raw.card_id.isin(val_cards)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_cards2 = np.random.choice(val_cards, 8000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid = df_raw.iloc[val_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idx2 = df_valid.index[df_valid.card_id.isin(val_cards2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((19249698, 91), (3433782, 91), 678088)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.shape, df_valid.shape, len(val_idx2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid_trn = df_valid[~df_valid.card_id.isin(val_cards2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2755694, 91), (3433782, 91), 678088)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid_trn.shape, df_valid.shape, len(val_idx2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3433782,)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_idx.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-41486728f3b1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrn_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf_raw\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m~\u001b[0m\u001b[0mdf_raw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcard_id\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_cards\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2916\u001b[0m         \u001b[0;31m# Do we have a (boolean) 1d indexer?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2917\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_bool_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2918\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_bool_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2919\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2920\u001b[0m         \u001b[0;31m# We are left with two options: a single key, and a collection of keys,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_getitem_bool_array\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2967\u001b[0m         \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_bool_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2968\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnonzero\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2969\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_take\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2970\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2971\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_take\u001b[0;34m(self, indices, axis, is_copy)\u001b[0m\n\u001b[1;32m   3357\u001b[0m         new_data = self._data.take(indices,\n\u001b[1;32m   3358\u001b[0m                                    \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_block_manager_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3359\u001b[0;31m                                    verify=True)\n\u001b[0m\u001b[1;32m   3360\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_constructor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__finalize__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3361\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36mtake\u001b[0;34m(self, indexer, axis, verify, convert)\u001b[0m\n\u001b[1;32m   1348\u001b[0m         \u001b[0mnew_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m         return self.reindex_indexer(new_axis=new_labels, indexer=indexer,\n\u001b[0;32m-> 1350\u001b[0;31m                                     axis=axis, allow_dups=True)\n\u001b[0m\u001b[1;32m   1351\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmerge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlsuffix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrsuffix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36mreindex_indexer\u001b[0;34m(self, new_axis, indexer, axis, fill_value, allow_dups, copy)\u001b[0m\n\u001b[1;32m   1233\u001b[0m             new_blocks = [blk.take_nd(indexer, axis=axis, fill_tuple=(\n\u001b[1;32m   1234\u001b[0m                 fill_value if fill_value is not None else blk.fill_value,))\n\u001b[0;32m-> 1235\u001b[0;31m                 for blk in self.blocks]\n\u001b[0m\u001b[1;32m   1236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1237\u001b[0m         \u001b[0mnew_axes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1233\u001b[0m             new_blocks = [blk.take_nd(indexer, axis=axis, fill_tuple=(\n\u001b[1;32m   1234\u001b[0m                 fill_value if fill_value is not None else blk.fill_value,))\n\u001b[0;32m-> 1235\u001b[0;31m                 for blk in self.blocks]\n\u001b[0m\u001b[1;32m   1236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1237\u001b[0m         \u001b[0mnew_axes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/internals/blocks.py\u001b[0m in \u001b[0;36mtake_nd\u001b[0;34m(self, indexer, axis, new_mgr_locs, fill_tuple)\u001b[0m\n\u001b[1;32m   1236\u001b[0m             \u001b[0mfill_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfill_tuple\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1237\u001b[0m             new_values = algos.take_nd(values, indexer, axis=axis,\n\u001b[0;32m-> 1238\u001b[0;31m                                        allow_fill=True, fill_value=fill_value)\n\u001b[0m\u001b[1;32m   1239\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1240\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnew_mgr_locs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/algorithms.py\u001b[0m in \u001b[0;36mtake_nd\u001b[0;34m(arr, indexer, axis, out, fill_value, mask_info, allow_fill)\u001b[0m\n\u001b[1;32m   1643\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'F'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1644\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1645\u001b[0;31m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1646\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1647\u001b[0m     func = _get_take_nd_function(arr.ndim, arr.dtype, out.dtype, axis=axis,\n",
      "\u001b[0;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "trn_idx = df_raw[~df_raw.card_id.isin(val_cards)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'trn_idx' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-264356339c73>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mval_idx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrn_idx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf_raw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'trn_idx' is not defined"
     ]
    }
   ],
   "source": [
    "val_idx.shape, trn_idx.shape, df_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['authorized_flag', 'card_id', 'installments', 'category_3', 'merchant_category_id', 'merchant_id',\n",
       "       'month_lag', 'purchase_amount', 'purchase_date', 'purchase_Year', 'purchase_Month', 'purchase_Week',\n",
       "       'purchase_Day', 'purchase_Dayofweek', 'purchase_Dayofyear', 'purchase_Is_month_end',\n",
       "       'purchase_Is_month_start', 'purchase_Is_quarter_end', 'purchase_Is_quarter_start',\n",
       "       'purchase_Is_year_end', 'purchase_Is_year_start', 'purchase_Hour', 'purchase_Minute',\n",
       "       'purchase_Second', 'purchase_Elapsed', 'purchased_on_weekend', 'purchased_on_weekday', 'month_diff',\n",
       "       'purchase_date_successive_diff', 'Christmas_Day_2017', 'Mothers_Day_2017', 'fathers_day_2017',\n",
       "       'Children_day_2017', 'Valentine_Day_2017', 'Black_Friday_2017', 'Mothers_Day_2018', 'duration',\n",
       "       'amount_month_ratio', 'merchant_address_id', 'numerical_range', 'merchant_rating',\n",
       "       'merchant_group_id', 'merchant_category_id_y', 'subsector_id', 'numerical_1', 'numerical_2',\n",
       "       'category_1', 'most_recent_sales_range', 'most_recent_purchases_range', 'avg_sales_lag3',\n",
       "       'avg_purchases_lag3', 'active_months_lag3', 'avg_sales_lag6', 'avg_purchases_lag6',\n",
       "       'active_months_lag6', 'avg_sales_lag12', 'avg_purchases_lag12', 'active_months_lag12', 'category_4',\n",
       "       'city_id', 'state_id', 'category_2', 'CLV', 'CLV_old', 'CLV_ratio', 'card_id_count_ratio',\n",
       "       'days_feature1', 'days_feature1_ratio', 'days_feature2', 'days_feature2_ratio', 'days_feature3',\n",
       "       'days_feature3_ratio', 'feature_1', 'feature_2', 'feature_3', 'feature_max', 'feature_mean',\n",
       "       'feature_min', 'feature_sum', 'feature_var', 'hist_first_buy', 'hist_last_buy', 'installments_ratio',\n",
       "       'month_diff_ratio', 'new_first_buy', 'new_last_buy', 'observed_elapsed_time', 'outliers',\n",
       "       'purchase_amount_sum_ratio', 'quarter', 'target'], dtype=object)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "authorized_flag                  category\n",
       "card_id                            object\n",
       "installments                      float64\n",
       "category_3                       category\n",
       "merchant_category_id             category\n",
       "merchant_id                        object\n",
       "month_lag                           int64\n",
       "purchase_amount                   float64\n",
       "purchase_date                     float64\n",
       "purchase_Year                    category\n",
       "purchase_Month                   category\n",
       "purchase_Week                    category\n",
       "purchase_Day                     category\n",
       "purchase_Dayofweek               category\n",
       "purchase_Dayofyear               category\n",
       "purchase_Is_month_end                bool\n",
       "purchase_Is_month_start              bool\n",
       "purchase_Is_quarter_end              bool\n",
       "purchase_Is_quarter_start            bool\n",
       "purchase_Is_year_end                 bool\n",
       "purchase_Is_year_start               bool\n",
       "purchase_Hour                    category\n",
       "purchase_Minute                  category\n",
       "purchase_Second                  category\n",
       "purchase_Elapsed                    int64\n",
       "purchased_on_weekend             category\n",
       "purchased_on_weekday             category\n",
       "month_diff                          int64\n",
       "purchase_date_successive_diff     float64\n",
       "Christmas_Day_2017               category\n",
       "                                   ...   \n",
       "category_2                       category\n",
       "CLV                               float64\n",
       "CLV_old                           float64\n",
       "CLV_ratio                         float64\n",
       "card_id_count_ratio               float64\n",
       "days_feature1                    category\n",
       "days_feature1_ratio              category\n",
       "days_feature2                    category\n",
       "days_feature2_ratio              category\n",
       "days_feature3                    category\n",
       "days_feature3_ratio              category\n",
       "feature_1                        category\n",
       "feature_2                        category\n",
       "feature_3                        category\n",
       "feature_max                      category\n",
       "feature_mean                     category\n",
       "feature_min                      category\n",
       "feature_sum                      category\n",
       "feature_var                      category\n",
       "hist_first_buy                   category\n",
       "hist_last_buy                    category\n",
       "installments_ratio                float64\n",
       "month_diff_ratio                  float64\n",
       "new_first_buy                    category\n",
       "new_last_buy                     category\n",
       "observed_elapsed_time            category\n",
       "outliers                            int64\n",
       "purchase_amount_sum_ratio         float64\n",
       "quarter                          category\n",
       "target                            float64\n",
       "Length: 91, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['authorized_flag', 'category_3', 'merchant_category_id', 'purchase_Year', 'purchase_Month',\n",
       "       'purchase_Week', 'purchase_Day', 'purchase_Dayofweek', 'purchase_Dayofyear', 'purchase_Hour',\n",
       "       'purchase_Minute', 'purchase_Second', 'purchased_on_weekend', 'purchased_on_weekday',\n",
       "       'Christmas_Day_2017', 'Mothers_Day_2017', 'fathers_day_2017', 'Children_day_2017',\n",
       "       'Valentine_Day_2017', 'Black_Friday_2017', 'Mothers_Day_2018', 'merchant_rating', 'merchant_group_id',\n",
       "       'subsector_id', 'category_1', 'most_recent_sales_range', 'most_recent_purchases_range',\n",
       "       'active_months_lag3', 'active_months_lag6', 'active_months_lag12', 'category_4', 'city_id',\n",
       "       'state_id', 'category_2', 'days_feature1', 'days_feature1_ratio', 'days_feature2',\n",
       "       'days_feature2_ratio', 'days_feature3', 'days_feature3_ratio', 'feature_1', 'feature_2', 'feature_3',\n",
       "       'feature_max', 'feature_mean', 'feature_min', 'feature_sum', 'feature_var', 'hist_first_buy',\n",
       "       'hist_last_buy', 'new_first_buy', 'new_last_buy', 'observed_elapsed_time', 'quarter'], dtype=object)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.select_dtypes(include=['category']).columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['authorized_flag', 'category_3', 'merchant_category_id', 'purchase_Year', 'purchase_Month',\n",
       "       'purchase_Week', 'purchase_Day', 'purchase_Dayofweek', 'purchase_Dayofyear', 'purchase_Hour',\n",
       "       'purchase_Minute', 'purchase_Second', 'purchased_on_weekend', 'purchased_on_weekday',\n",
       "       'Christmas_Day_2017', 'Mothers_Day_2017', 'fathers_day_2017', 'Children_day_2017',\n",
       "       'Valentine_Day_2017', 'Black_Friday_2017', 'Mothers_Day_2018', 'merchant_rating', 'merchant_group_id',\n",
       "       'subsector_id', 'category_1', 'most_recent_sales_range', 'most_recent_purchases_range',\n",
       "       'active_months_lag3', 'active_months_lag6', 'active_months_lag12', 'category_4', 'city_id',\n",
       "       'state_id', 'category_2', 'days_feature1', 'days_feature1_ratio', 'days_feature2',\n",
       "       'days_feature2_ratio', 'days_feature3', 'days_feature3_ratio', 'feature_1', 'feature_2', 'feature_3',\n",
       "       'feature_max', 'feature_mean', 'feature_min', 'feature_sum', 'feature_var', 'hist_first_buy',\n",
       "       'hist_last_buy', 'new_first_buy', 'new_last_buy', 'observed_elapsed_time', 'quarter'], dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.select_dtypes(include=['category']).columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_flds = df_raw.select_dtypes(include=['category']).columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_flds = cat_flds.tolist() + ['card_id', 'merchant_id', 'merchant_address_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>authorized_flag</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <td>C_ID_da2090f28e</td>\n",
       "      <td>C_ID_efced389a0</td>\n",
       "      <td>C_ID_479fd6392a</td>\n",
       "      <td>C_ID_79296614ed</td>\n",
       "      <td>C_ID_f43ae18887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_category_id</th>\n",
       "      <td>623</td>\n",
       "      <td>842</td>\n",
       "      <td>839</td>\n",
       "      <td>360</td>\n",
       "      <td>511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_id</th>\n",
       "      <td>M_ID_f001319a61</td>\n",
       "      <td>M_ID_18038b5ae7</td>\n",
       "      <td>M_ID_e5374dabc0</td>\n",
       "      <td>M_ID_257157f6ee</td>\n",
       "      <td>M_ID_00a6ca8a8a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag</th>\n",
       "      <td>-11</td>\n",
       "      <td>-12</td>\n",
       "      <td>-1</td>\n",
       "      <td>-3</td>\n",
       "      <td>-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount</th>\n",
       "      <td>40</td>\n",
       "      <td>120</td>\n",
       "      <td>6</td>\n",
       "      <td>108.5</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date</th>\n",
       "      <td>1.48323e+09</td>\n",
       "      <td>1.48323e+09</td>\n",
       "      <td>1.48323e+09</td>\n",
       "      <td>1.48323e+09</td>\n",
       "      <td>1.48323e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Year</th>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Month</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Week</th>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Day</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Dayofweek</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Dayofyear</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_month_end</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_month_start</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_quarter_end</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_quarter_start</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_year_end</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_year_start</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Hour</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Minute</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Second</th>\n",
       "      <td>8</td>\n",
       "      <td>59</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Elapsed</th>\n",
       "      <td>1483228808</td>\n",
       "      <td>1483228859</td>\n",
       "      <td>1483228923</td>\n",
       "      <td>1483228934</td>\n",
       "      <td>1483228945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekday</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff</th>\n",
       "      <td>14</td>\n",
       "      <td>13</td>\n",
       "      <td>24</td>\n",
       "      <td>22</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_successive_diff</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Christmas_Day_2017</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mothers_Day_2017</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fathers_day_2017</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Children_day_2017</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Valentine_Day_2017</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Black_Friday_2017</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mothers_Day_2018</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>duration</th>\n",
       "      <td>560</td>\n",
       "      <td>1560</td>\n",
       "      <td>144</td>\n",
       "      <td>2387</td>\n",
       "      <td>468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>amount_month_ratio</th>\n",
       "      <td>2.85714</td>\n",
       "      <td>9.23077</td>\n",
       "      <td>0.25</td>\n",
       "      <td>4.93182</td>\n",
       "      <td>3.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_address_id</th>\n",
       "      <td>M_ID_f001319a61_623_4_0_9_1</td>\n",
       "      <td>M_ID_18038b5ae7_842_37_76_2_3</td>\n",
       "      <td>M_ID_e5374dabc0_839_29_0_0_0</td>\n",
       "      <td>M_ID_257157f6ee_705_33_212_4_4</td>\n",
       "      <td>M_ID_00a6ca8a8a_511_7_0_0_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>numerical_range</th>\n",
       "      <td>0</td>\n",
       "      <td>189</td>\n",
       "      <td>0</td>\n",
       "      <td>1689</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_rating</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_group_id</th>\n",
       "      <td>2119</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_category_id_y</th>\n",
       "      <td>623</td>\n",
       "      <td>842</td>\n",
       "      <td>839</td>\n",
       "      <td>705</td>\n",
       "      <td>511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsector_id</th>\n",
       "      <td>4</td>\n",
       "      <td>37</td>\n",
       "      <td>29</td>\n",
       "      <td>33</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>numerical_1</th>\n",
       "      <td>-0.0574706</td>\n",
       "      <td>0.884445</td>\n",
       "      <td>-0.0574706</td>\n",
       "      <td>8.32062</td>\n",
       "      <td>-0.0475558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>numerical_2</th>\n",
       "      <td>-0.0574706</td>\n",
       "      <td>0.87453</td>\n",
       "      <td>-0.0574706</td>\n",
       "      <td>8.31071</td>\n",
       "      <td>-0.0574706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_1</th>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>most_recent_sales_range</th>\n",
       "      <td>B</td>\n",
       "      <td>B</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>most_recent_purchases_range</th>\n",
       "      <td>B</td>\n",
       "      <td>B</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_sales_lag3</th>\n",
       "      <td>1.02</td>\n",
       "      <td>1.11</td>\n",
       "      <td>0.94</td>\n",
       "      <td>1.19</td>\n",
       "      <td>6.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_purchases_lag3</th>\n",
       "      <td>1.02696</td>\n",
       "      <td>1.09963</td>\n",
       "      <td>0.911346</td>\n",
       "      <td>1.16483</td>\n",
       "      <td>12.7051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>active_months_lag3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_sales_lag6</th>\n",
       "      <td>1</td>\n",
       "      <td>1.28</td>\n",
       "      <td>0.81</td>\n",
       "      <td>1.27</td>\n",
       "      <td>8.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_purchases_lag6</th>\n",
       "      <td>1.01143</td>\n",
       "      <td>1.2036</td>\n",
       "      <td>0.777814</td>\n",
       "      <td>1.2324</td>\n",
       "      <td>15.8558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>active_months_lag6</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_sales_lag12</th>\n",
       "      <td>1.07</td>\n",
       "      <td>1.48</td>\n",
       "      <td>0.62</td>\n",
       "      <td>1.35</td>\n",
       "      <td>8.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_purchases_lag12</th>\n",
       "      <td>1.06246</td>\n",
       "      <td>1.33641</td>\n",
       "      <td>0.583477</td>\n",
       "      <td>1.2875</td>\n",
       "      <td>16.2198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>active_months_lag12</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_4</th>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city_id</th>\n",
       "      <td>-1</td>\n",
       "      <td>76</td>\n",
       "      <td>-1</td>\n",
       "      <td>212</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_id</th>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLV</th>\n",
       "      <td>843.02</td>\n",
       "      <td>6449.36</td>\n",
       "      <td>134.591</td>\n",
       "      <td>960.884</td>\n",
       "      <td>1048.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLV_old</th>\n",
       "      <td>216088</td>\n",
       "      <td>277773</td>\n",
       "      <td>70.3312</td>\n",
       "      <td>11599.4</td>\n",
       "      <td>620636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLV_ratio</th>\n",
       "      <td>0.00390128</td>\n",
       "      <td>0.0232181</td>\n",
       "      <td>0.00560192</td>\n",
       "      <td>0.0828392</td>\n",
       "      <td>0.0016886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id_count_ratio</th>\n",
       "      <td>0.0451389</td>\n",
       "      <td>0.125</td>\n",
       "      <td>0.0789474</td>\n",
       "      <td>0.1875</td>\n",
       "      <td>0.0445205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_feature1</th>\n",
       "      <td>3747</td>\n",
       "      <td>427</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1641</td>\n",
       "      <td>910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_feature1_ratio</th>\n",
       "      <td>0.00240192</td>\n",
       "      <td>0.00234192</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00548446</td>\n",
       "      <td>0.0043956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_feature2</th>\n",
       "      <td>3747</td>\n",
       "      <td>1281</td>\n",
       "      <td>NaN</td>\n",
       "      <td>547</td>\n",
       "      <td>910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_feature2_ratio</th>\n",
       "      <td>0.00240192</td>\n",
       "      <td>0.00702576</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00182815</td>\n",
       "      <td>0.0043956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_feature3</th>\n",
       "      <td>1249</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>547</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_feature3_ratio</th>\n",
       "      <td>0.000800641</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00182815</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1</th>\n",
       "      <td>0.0104794</td>\n",
       "      <td>0.00805849</td>\n",
       "      <td>0.00805849</td>\n",
       "      <td>0.0104794</td>\n",
       "      <td>0.0106099</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2</th>\n",
       "      <td>0.0141664</td>\n",
       "      <td>0.0141664</td>\n",
       "      <td>0.0113848</td>\n",
       "      <td>0.0113848</td>\n",
       "      <td>0.00875212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_3</th>\n",
       "      <td>0.0114275</td>\n",
       "      <td>0.0102828</td>\n",
       "      <td>0.0102828</td>\n",
       "      <td>0.0114275</td>\n",
       "      <td>0.0102828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_max</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_mean</th>\n",
       "      <td>2.33333</td>\n",
       "      <td>1.33333</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.66667</td>\n",
       "      <td>1.33333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_min</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_sum</th>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_var</th>\n",
       "      <td>1.1547</td>\n",
       "      <td>1.52753</td>\n",
       "      <td>0.57735</td>\n",
       "      <td>1.1547</td>\n",
       "      <td>1.1547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist_first_buy</th>\n",
       "      <td>884</td>\n",
       "      <td>31</td>\n",
       "      <td>92</td>\n",
       "      <td>427</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist_last_buy</th>\n",
       "      <td>1247</td>\n",
       "      <td>412</td>\n",
       "      <td>149</td>\n",
       "      <td>539</td>\n",
       "      <td>454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_ratio</th>\n",
       "      <td>0.0408163</td>\n",
       "      <td>0.072327</td>\n",
       "      <td>0.0408163</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0408163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff_ratio</th>\n",
       "      <td>0.976801</td>\n",
       "      <td>1.00398</td>\n",
       "      <td>1.00117</td>\n",
       "      <td>1.00081</td>\n",
       "      <td>0.998291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_first_buy</th>\n",
       "      <td>1263</td>\n",
       "      <td>445</td>\n",
       "      <td>NaN</td>\n",
       "      <td>550</td>\n",
       "      <td>455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_last_buy</th>\n",
       "      <td>1289</td>\n",
       "      <td>478</td>\n",
       "      <td>NaN</td>\n",
       "      <td>592</td>\n",
       "      <td>513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>observed_elapsed_time</th>\n",
       "      <td>1249</td>\n",
       "      <td>427</td>\n",
       "      <td>NaN</td>\n",
       "      <td>547</td>\n",
       "      <td>455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>outliers</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_sum_ratio</th>\n",
       "      <td>0.0844233</td>\n",
       "      <td>0.186483</td>\n",
       "      <td>0.0712785</td>\n",
       "      <td>0.442168</td>\n",
       "      <td>0.0378638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>quarter</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>-1.99163</td>\n",
       "      <td>-1.84033</td>\n",
       "      <td>0.89228</td>\n",
       "      <td>-1.34976</td>\n",
       "      <td>-0.964822</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                         0  \\\n",
       "authorized_flag                                          1   \n",
       "card_id                                    C_ID_da2090f28e   \n",
       "installments                                             0   \n",
       "category_3                                               0   \n",
       "merchant_category_id                                   623   \n",
       "merchant_id                                M_ID_f001319a61   \n",
       "month_lag                                              -11   \n",
       "purchase_amount                                         40   \n",
       "purchase_date                                  1.48323e+09   \n",
       "purchase_Year                                         2017   \n",
       "purchase_Month                                           1   \n",
       "purchase_Week                                           52   \n",
       "purchase_Day                                             1   \n",
       "purchase_Dayofweek                                       6   \n",
       "purchase_Dayofyear                                       1   \n",
       "purchase_Is_month_end                                False   \n",
       "purchase_Is_month_start                               True   \n",
       "purchase_Is_quarter_end                              False   \n",
       "purchase_Is_quarter_start                             True   \n",
       "purchase_Is_year_end                                 False   \n",
       "purchase_Is_year_start                                True   \n",
       "purchase_Hour                                            0   \n",
       "purchase_Minute                                          0   \n",
       "purchase_Second                                          8   \n",
       "purchase_Elapsed                                1483228808   \n",
       "purchased_on_weekend                                     1   \n",
       "purchased_on_weekday                                     0   \n",
       "month_diff                                              14   \n",
       "purchase_date_successive_diff                            0   \n",
       "Christmas_Day_2017                                       0   \n",
       "Mothers_Day_2017                                         0   \n",
       "fathers_day_2017                                         0   \n",
       "Children_day_2017                                        0   \n",
       "Valentine_Day_2017                                       0   \n",
       "Black_Friday_2017                                        0   \n",
       "Mothers_Day_2018                                         0   \n",
       "duration                                               560   \n",
       "amount_month_ratio                                 2.85714   \n",
       "merchant_address_id            M_ID_f001319a61_623_4_0_9_1   \n",
       "numerical_range                                          0   \n",
       "merchant_rating                                         12   \n",
       "merchant_group_id                                     2119   \n",
       "merchant_category_id_y                                 623   \n",
       "subsector_id                                             4   \n",
       "numerical_1                                     -0.0574706   \n",
       "numerical_2                                     -0.0574706   \n",
       "category_1                                               N   \n",
       "most_recent_sales_range                                  B   \n",
       "most_recent_purchases_range                              B   \n",
       "avg_sales_lag3                                        1.02   \n",
       "avg_purchases_lag3                                 1.02696   \n",
       "active_months_lag3                                       3   \n",
       "avg_sales_lag6                                           1   \n",
       "avg_purchases_lag6                                 1.01143   \n",
       "active_months_lag6                                       6   \n",
       "avg_sales_lag12                                       1.07   \n",
       "avg_purchases_lag12                                1.06246   \n",
       "active_months_lag12                                     12   \n",
       "category_4                                               N   \n",
       "city_id                                                 -1   \n",
       "state_id                                                 9   \n",
       "category_2                                               1   \n",
       "CLV                                                 843.02   \n",
       "CLV_old                                             216088   \n",
       "CLV_ratio                                       0.00390128   \n",
       "card_id_count_ratio                              0.0451389   \n",
       "days_feature1                                         3747   \n",
       "days_feature1_ratio                             0.00240192   \n",
       "days_feature2                                         3747   \n",
       "days_feature2_ratio                             0.00240192   \n",
       "days_feature3                                         1249   \n",
       "days_feature3_ratio                            0.000800641   \n",
       "feature_1                                        0.0104794   \n",
       "feature_2                                        0.0141664   \n",
       "feature_3                                        0.0114275   \n",
       "feature_max                                              3   \n",
       "feature_mean                                       2.33333   \n",
       "feature_min                                              1   \n",
       "feature_sum                                              7   \n",
       "feature_var                                         1.1547   \n",
       "hist_first_buy                                         884   \n",
       "hist_last_buy                                         1247   \n",
       "installments_ratio                               0.0408163   \n",
       "month_diff_ratio                                  0.976801   \n",
       "new_first_buy                                         1263   \n",
       "new_last_buy                                          1289   \n",
       "observed_elapsed_time                                 1249   \n",
       "outliers                                                 0   \n",
       "purchase_amount_sum_ratio                        0.0844233   \n",
       "quarter                                                  3   \n",
       "target                                            -1.99163   \n",
       "\n",
       "                                                           1  \\\n",
       "authorized_flag                                            1   \n",
       "card_id                                      C_ID_efced389a0   \n",
       "installments                                               1   \n",
       "category_3                                                 1   \n",
       "merchant_category_id                                     842   \n",
       "merchant_id                                  M_ID_18038b5ae7   \n",
       "month_lag                                                -12   \n",
       "purchase_amount                                          120   \n",
       "purchase_date                                    1.48323e+09   \n",
       "purchase_Year                                           2017   \n",
       "purchase_Month                                             1   \n",
       "purchase_Week                                             52   \n",
       "purchase_Day                                               1   \n",
       "purchase_Dayofweek                                         6   \n",
       "purchase_Dayofyear                                         1   \n",
       "purchase_Is_month_end                                  False   \n",
       "purchase_Is_month_start                                 True   \n",
       "purchase_Is_quarter_end                                False   \n",
       "purchase_Is_quarter_start                               True   \n",
       "purchase_Is_year_end                                   False   \n",
       "purchase_Is_year_start                                  True   \n",
       "purchase_Hour                                              0   \n",
       "purchase_Minute                                            0   \n",
       "purchase_Second                                           59   \n",
       "purchase_Elapsed                                  1483228859   \n",
       "purchased_on_weekend                                       1   \n",
       "purchased_on_weekday                                       0   \n",
       "month_diff                                                13   \n",
       "purchase_date_successive_diff                              0   \n",
       "Christmas_Day_2017                                         0   \n",
       "Mothers_Day_2017                                           0   \n",
       "fathers_day_2017                                           0   \n",
       "Children_day_2017                                          0   \n",
       "Valentine_Day_2017                                         0   \n",
       "Black_Friday_2017                                          0   \n",
       "Mothers_Day_2018                                           0   \n",
       "duration                                                1560   \n",
       "amount_month_ratio                                   9.23077   \n",
       "merchant_address_id            M_ID_18038b5ae7_842_37_76_2_3   \n",
       "numerical_range                                          189   \n",
       "merchant_rating                                           12   \n",
       "merchant_group_id                                         35   \n",
       "merchant_category_id_y                                   842   \n",
       "subsector_id                                              37   \n",
       "numerical_1                                         0.884445   \n",
       "numerical_2                                          0.87453   \n",
       "category_1                                                 N   \n",
       "most_recent_sales_range                                    B   \n",
       "most_recent_purchases_range                                B   \n",
       "avg_sales_lag3                                          1.11   \n",
       "avg_purchases_lag3                                   1.09963   \n",
       "active_months_lag3                                         3   \n",
       "avg_sales_lag6                                          1.28   \n",
       "avg_purchases_lag6                                    1.2036   \n",
       "active_months_lag6                                         6   \n",
       "avg_sales_lag12                                         1.48   \n",
       "avg_purchases_lag12                                  1.33641   \n",
       "active_months_lag12                                       12   \n",
       "category_4                                                 Y   \n",
       "city_id                                                   76   \n",
       "state_id                                                   2   \n",
       "category_2                                                 3   \n",
       "CLV                                                  6449.36   \n",
       "CLV_old                                               277773   \n",
       "CLV_ratio                                          0.0232181   \n",
       "card_id_count_ratio                                    0.125   \n",
       "days_feature1                                            427   \n",
       "days_feature1_ratio                               0.00234192   \n",
       "days_feature2                                           1281   \n",
       "days_feature2_ratio                               0.00702576   \n",
       "days_feature3                                              0   \n",
       "days_feature3_ratio                                        0   \n",
       "feature_1                                         0.00805849   \n",
       "feature_2                                          0.0141664   \n",
       "feature_3                                          0.0102828   \n",
       "feature_max                                                3   \n",
       "feature_mean                                         1.33333   \n",
       "feature_min                                                0   \n",
       "feature_sum                                                4   \n",
       "feature_var                                          1.52753   \n",
       "hist_first_buy                                            31   \n",
       "hist_last_buy                                            412   \n",
       "installments_ratio                                  0.072327   \n",
       "month_diff_ratio                                     1.00398   \n",
       "new_first_buy                                            445   \n",
       "new_last_buy                                             478   \n",
       "observed_elapsed_time                                    427   \n",
       "outliers                                                   0   \n",
       "purchase_amount_sum_ratio                           0.186483   \n",
       "quarter                                                    4   \n",
       "target                                              -1.84033   \n",
       "\n",
       "                                                          2  \\\n",
       "authorized_flag                                           1   \n",
       "card_id                                     C_ID_479fd6392a   \n",
       "installments                                              1   \n",
       "category_3                                                1   \n",
       "merchant_category_id                                    839   \n",
       "merchant_id                                 M_ID_e5374dabc0   \n",
       "month_lag                                                -1   \n",
       "purchase_amount                                           6   \n",
       "purchase_date                                   1.48323e+09   \n",
       "purchase_Year                                          2017   \n",
       "purchase_Month                                            1   \n",
       "purchase_Week                                            52   \n",
       "purchase_Day                                              1   \n",
       "purchase_Dayofweek                                        6   \n",
       "purchase_Dayofyear                                        1   \n",
       "purchase_Is_month_end                                 False   \n",
       "purchase_Is_month_start                                True   \n",
       "purchase_Is_quarter_end                               False   \n",
       "purchase_Is_quarter_start                              True   \n",
       "purchase_Is_year_end                                  False   \n",
       "purchase_Is_year_start                                 True   \n",
       "purchase_Hour                                             0   \n",
       "purchase_Minute                                           2   \n",
       "purchase_Second                                           3   \n",
       "purchase_Elapsed                                 1483228923   \n",
       "purchased_on_weekend                                      1   \n",
       "purchased_on_weekday                                      0   \n",
       "month_diff                                               24   \n",
       "purchase_date_successive_diff                             0   \n",
       "Christmas_Day_2017                                        0   \n",
       "Mothers_Day_2017                                          0   \n",
       "fathers_day_2017                                          0   \n",
       "Children_day_2017                                         0   \n",
       "Valentine_Day_2017                                        0   \n",
       "Black_Friday_2017                                         0   \n",
       "Mothers_Day_2018                                          0   \n",
       "duration                                                144   \n",
       "amount_month_ratio                                     0.25   \n",
       "merchant_address_id            M_ID_e5374dabc0_839_29_0_0_0   \n",
       "numerical_range                                           0   \n",
       "merchant_rating                                           6   \n",
       "merchant_group_id                                        35   \n",
       "merchant_category_id_y                                  839   \n",
       "subsector_id                                             29   \n",
       "numerical_1                                      -0.0574706   \n",
       "numerical_2                                      -0.0574706   \n",
       "category_1                                                Y   \n",
       "most_recent_sales_range                                   A   \n",
       "most_recent_purchases_range                               A   \n",
       "avg_sales_lag3                                         0.94   \n",
       "avg_purchases_lag3                                 0.911346   \n",
       "active_months_lag3                                        3   \n",
       "avg_sales_lag6                                         0.81   \n",
       "avg_purchases_lag6                                 0.777814   \n",
       "active_months_lag6                                        6   \n",
       "avg_sales_lag12                                        0.62   \n",
       "avg_purchases_lag12                                0.583477   \n",
       "active_months_lag12                                      12   \n",
       "category_4                                                Y   \n",
       "city_id                                                  -1   \n",
       "state_id                                                 -1   \n",
       "category_2                                              NaN   \n",
       "CLV                                                 134.591   \n",
       "CLV_old                                             70.3312   \n",
       "CLV_ratio                                        0.00560192   \n",
       "card_id_count_ratio                               0.0789474   \n",
       "days_feature1                                           NaN   \n",
       "days_feature1_ratio                                     NaN   \n",
       "days_feature2                                           NaN   \n",
       "days_feature2_ratio                                     NaN   \n",
       "days_feature3                                           NaN   \n",
       "days_feature3_ratio                                     NaN   \n",
       "feature_1                                        0.00805849   \n",
       "feature_2                                         0.0113848   \n",
       "feature_3                                         0.0102828   \n",
       "feature_max                                               1   \n",
       "feature_mean                                       0.666667   \n",
       "feature_min                                               0   \n",
       "feature_sum                                               2   \n",
       "feature_var                                         0.57735   \n",
       "hist_first_buy                                           92   \n",
       "hist_last_buy                                           149   \n",
       "installments_ratio                                0.0408163   \n",
       "month_diff_ratio                                    1.00117   \n",
       "new_first_buy                                           NaN   \n",
       "new_last_buy                                            NaN   \n",
       "observed_elapsed_time                                   NaN   \n",
       "outliers                                                  0   \n",
       "purchase_amount_sum_ratio                         0.0712785   \n",
       "quarter                                                   4   \n",
       "target                                              0.89228   \n",
       "\n",
       "                                                            3  \\\n",
       "authorized_flag                                             0   \n",
       "card_id                                       C_ID_79296614ed   \n",
       "installments                                                0   \n",
       "category_3                                                  0   \n",
       "merchant_category_id                                      360   \n",
       "merchant_id                                   M_ID_257157f6ee   \n",
       "month_lag                                                  -3   \n",
       "purchase_amount                                         108.5   \n",
       "purchase_date                                     1.48323e+09   \n",
       "purchase_Year                                            2017   \n",
       "purchase_Month                                              1   \n",
       "purchase_Week                                              52   \n",
       "purchase_Day                                                1   \n",
       "purchase_Dayofweek                                          6   \n",
       "purchase_Dayofyear                                          1   \n",
       "purchase_Is_month_end                                   False   \n",
       "purchase_Is_month_start                                  True   \n",
       "purchase_Is_quarter_end                                 False   \n",
       "purchase_Is_quarter_start                                True   \n",
       "purchase_Is_year_end                                    False   \n",
       "purchase_Is_year_start                                   True   \n",
       "purchase_Hour                                               0   \n",
       "purchase_Minute                                             2   \n",
       "purchase_Second                                            14   \n",
       "purchase_Elapsed                                   1483228934   \n",
       "purchased_on_weekend                                        1   \n",
       "purchased_on_weekday                                        0   \n",
       "month_diff                                                 22   \n",
       "purchase_date_successive_diff                               0   \n",
       "Christmas_Day_2017                                          0   \n",
       "Mothers_Day_2017                                            0   \n",
       "fathers_day_2017                                            0   \n",
       "Children_day_2017                                           0   \n",
       "Valentine_Day_2017                                          0   \n",
       "Black_Friday_2017                                           0   \n",
       "Mothers_Day_2018                                            0   \n",
       "duration                                                 2387   \n",
       "amount_month_ratio                                    4.93182   \n",
       "merchant_address_id            M_ID_257157f6ee_705_33_212_4_4   \n",
       "numerical_range                                          1689   \n",
       "merchant_rating                                            10   \n",
       "merchant_group_id                                          35   \n",
       "merchant_category_id_y                                    705   \n",
       "subsector_id                                               33   \n",
       "numerical_1                                           8.32062   \n",
       "numerical_2                                           8.31071   \n",
       "category_1                                                  N   \n",
       "most_recent_sales_range                                     A   \n",
       "most_recent_purchases_range                                 A   \n",
       "avg_sales_lag3                                           1.19   \n",
       "avg_purchases_lag3                                    1.16483   \n",
       "active_months_lag3                                          3   \n",
       "avg_sales_lag6                                           1.27   \n",
       "avg_purchases_lag6                                     1.2324   \n",
       "active_months_lag6                                          6   \n",
       "avg_sales_lag12                                          1.35   \n",
       "avg_purchases_lag12                                    1.2875   \n",
       "active_months_lag12                                        12   \n",
       "category_4                                                  Y   \n",
       "city_id                                                   212   \n",
       "state_id                                                    4   \n",
       "category_2                                                  4   \n",
       "CLV                                                   960.884   \n",
       "CLV_old                                               11599.4   \n",
       "CLV_ratio                                           0.0828392   \n",
       "card_id_count_ratio                                    0.1875   \n",
       "days_feature1                                            1641   \n",
       "days_feature1_ratio                                0.00548446   \n",
       "days_feature2                                             547   \n",
       "days_feature2_ratio                                0.00182815   \n",
       "days_feature3                                             547   \n",
       "days_feature3_ratio                                0.00182815   \n",
       "feature_1                                           0.0104794   \n",
       "feature_2                                           0.0113848   \n",
       "feature_3                                           0.0114275   \n",
       "feature_max                                                 3   \n",
       "feature_mean                                          1.66667   \n",
       "feature_min                                                 1   \n",
       "feature_sum                                                 5   \n",
       "feature_var                                            1.1547   \n",
       "hist_first_buy                                            427   \n",
       "hist_last_buy                                             539   \n",
       "installments_ratio                                          0   \n",
       "month_diff_ratio                                      1.00081   \n",
       "new_first_buy                                             550   \n",
       "new_last_buy                                              592   \n",
       "observed_elapsed_time                                     547   \n",
       "outliers                                                    0   \n",
       "purchase_amount_sum_ratio                            0.442168   \n",
       "quarter                                                     4   \n",
       "target                                               -1.34976   \n",
       "\n",
       "                                                         4  \n",
       "authorized_flag                                          1  \n",
       "card_id                                    C_ID_f43ae18887  \n",
       "installments                                             0  \n",
       "category_3                                               0  \n",
       "merchant_category_id                                   511  \n",
       "merchant_id                                M_ID_00a6ca8a8a  \n",
       "month_lag                                              -13  \n",
       "purchase_amount                                         39  \n",
       "purchase_date                                  1.48323e+09  \n",
       "purchase_Year                                         2017  \n",
       "purchase_Month                                           1  \n",
       "purchase_Week                                           52  \n",
       "purchase_Day                                             1  \n",
       "purchase_Dayofweek                                       6  \n",
       "purchase_Dayofyear                                       1  \n",
       "purchase_Is_month_end                                False  \n",
       "purchase_Is_month_start                               True  \n",
       "purchase_Is_quarter_end                              False  \n",
       "purchase_Is_quarter_start                             True  \n",
       "purchase_Is_year_end                                 False  \n",
       "purchase_Is_year_start                                True  \n",
       "purchase_Hour                                            0  \n",
       "purchase_Minute                                          2  \n",
       "purchase_Second                                         25  \n",
       "purchase_Elapsed                                1483228945  \n",
       "purchased_on_weekend                                     1  \n",
       "purchased_on_weekday                                     0  \n",
       "month_diff                                              12  \n",
       "purchase_date_successive_diff                            0  \n",
       "Christmas_Day_2017                                       0  \n",
       "Mothers_Day_2017                                         0  \n",
       "fathers_day_2017                                         0  \n",
       "Children_day_2017                                        0  \n",
       "Valentine_Day_2017                                       0  \n",
       "Black_Friday_2017                                        0  \n",
       "Mothers_Day_2018                                         0  \n",
       "duration                                               468  \n",
       "amount_month_ratio                                    3.25  \n",
       "merchant_address_id            M_ID_00a6ca8a8a_511_7_0_0_0  \n",
       "numerical_range                                          1  \n",
       "merchant_rating                                         18  \n",
       "merchant_group_id                                       35  \n",
       "merchant_category_id_y                                 511  \n",
       "subsector_id                                             7  \n",
       "numerical_1                                     -0.0475558  \n",
       "numerical_2                                     -0.0574706  \n",
       "category_1                                               Y  \n",
       "most_recent_sales_range                                  E  \n",
       "most_recent_purchases_range                              E  \n",
       "avg_sales_lag3                                        6.93  \n",
       "avg_purchases_lag3                                 12.7051  \n",
       "active_months_lag3                                       3  \n",
       "avg_sales_lag6                                        8.42  \n",
       "avg_purchases_lag6                                 15.8558  \n",
       "active_months_lag6                                       6  \n",
       "avg_sales_lag12                                       8.57  \n",
       "avg_purchases_lag12                                16.2198  \n",
       "active_months_lag12                                      7  \n",
       "category_4                                               Y  \n",
       "city_id                                                 -1  \n",
       "state_id                                                -1  \n",
       "category_2                                             NaN  \n",
       "CLV                                                1048.01  \n",
       "CLV_old                                             620636  \n",
       "CLV_ratio                                        0.0016886  \n",
       "card_id_count_ratio                              0.0445205  \n",
       "days_feature1                                          910  \n",
       "days_feature1_ratio                              0.0043956  \n",
       "days_feature2                                          910  \n",
       "days_feature2_ratio                              0.0043956  \n",
       "days_feature3                                            0  \n",
       "days_feature3_ratio                                      0  \n",
       "feature_1                                        0.0106099  \n",
       "feature_2                                       0.00875212  \n",
       "feature_3                                        0.0102828  \n",
       "feature_max                                              2  \n",
       "feature_mean                                       1.33333  \n",
       "feature_min                                              0  \n",
       "feature_sum                                              4  \n",
       "feature_var                                         1.1547  \n",
       "hist_first_buy                                          31  \n",
       "hist_last_buy                                          454  \n",
       "installments_ratio                               0.0408163  \n",
       "month_diff_ratio                                  0.998291  \n",
       "new_first_buy                                          455  \n",
       "new_last_buy                                           513  \n",
       "observed_elapsed_time                                  455  \n",
       "outliers                                                 0  \n",
       "purchase_amount_sum_ratio                        0.0378638  \n",
       "quarter                                                  4  \n",
       "target                                           -0.964822  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((19249698, 91), (11825701, 91))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "bool_flds = df_raw.select_dtypes(include=['bool']).columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_flds = cat_flds + bool_flds.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "cont_flds = [n for n in df_raw.columns if n not in cat_flds and n!= 'outliers']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# df_raw = df_raw[cat_flds+cont_flds]\n",
    "# df, y, nas, mapper_t = proc_df(df_raw, 'target', do_scale=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "for c in bool_flds:\n",
    "    df_valid[c] = df_valid[c].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[n for n in df_valid.drop(cat_flds,axis=1).columns if not is_numeric_dtype(df_valid[n])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "for c in cat_flds: df_valid[c] = df_valid[c].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "df_raw = df_raw[cat_flds+cont_flds+[dep]]\n",
    "df, y, nas, mapper_t = proc_df(df_raw, 'target', do_scale=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2772190, 91), (3450901, 91), 678711)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid_trn.shape, df_valid.shape, len(val_idx2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_val = df_valid['target']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### proc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sorted = df_valid.sort_values(['card_id', 'purchase_date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sorted.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sorted.drop('index', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((3433782, 91), (3433782, 91))"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid.shape, df_sorted.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min, sys: 3.28 s, total: 1min 4s\n",
      "Wall time: 20.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df,y, nas, mapper_t = proc_df(df_sorted, 'target', do_scale=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'mapper_t' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'mapper_t' is not defined"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_test_copy = df_test[cat_flds+cont_flds+[dep]].copy()\n",
    "df_t, _, nas, mapper = proc_df(df_test_copy,'target', do_scale=True, mapper=mapper_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3433782, 91)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(x,y): return math.sqrt(((x-y)**2).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'authorized_flag': 3,\n",
       " 'category_3': 4,\n",
       " 'merchant_category_id': 328,\n",
       " 'purchase_Year': 3,\n",
       " 'purchase_Month': 13,\n",
       " 'purchase_Week': 53,\n",
       " 'purchase_Day': 32,\n",
       " 'purchase_Dayofweek': 8,\n",
       " 'purchase_Dayofyear': 366,\n",
       " 'purchase_Hour': 25,\n",
       " 'purchase_Minute': 61,\n",
       " 'purchase_Second': 61,\n",
       " 'purchased_on_weekend': 3,\n",
       " 'purchased_on_weekday': 3,\n",
       " 'Christmas_Day_2017': 101,\n",
       " 'Mothers_Day_2017': 101,\n",
       " 'fathers_day_2017': 101,\n",
       " 'Children_day_2017': 101,\n",
       " 'Valentine_Day_2017': 101,\n",
       " 'Black_Friday_2017': 101,\n",
       " 'Mothers_Day_2018': 29,\n",
       " 'merchant_rating': 17,\n",
       " 'merchant_group_id': 109392,\n",
       " 'subsector_id': 42,\n",
       " 'category_1': 3,\n",
       " 'most_recent_sales_range': 6,\n",
       " 'most_recent_purchases_range': 6,\n",
       " 'active_months_lag3': 4,\n",
       " 'active_months_lag6': 7,\n",
       " 'active_months_lag12': 13,\n",
       " 'category_4': 3,\n",
       " 'city_id': 272,\n",
       " 'state_id': 26,\n",
       " 'category_2': 6,\n",
       " 'days_feature1': 644,\n",
       " 'days_feature1_ratio': 647,\n",
       " 'days_feature2': 488,\n",
       " 'days_feature2_ratio': 505,\n",
       " 'days_feature3': 233,\n",
       " 'days_feature3_ratio': 234,\n",
       " 'feature_1': 6,\n",
       " 'feature_2': 4,\n",
       " 'feature_3': 3,\n",
       " 'feature_max': 6,\n",
       " 'feature_mean': 8,\n",
       " 'feature_min': 3,\n",
       " 'feature_sum': 8,\n",
       " 'feature_var': 10,\n",
       " 'hist_first_buy': 2005,\n",
       " 'hist_last_buy': 1955,\n",
       " 'new_first_buy': 1963,\n",
       " 'new_last_buy': 1991,\n",
       " 'observed_elapsed_time': 233,\n",
       " 'quarter': 5,\n",
       " 'card_id': 36248,\n",
       " 'merchant_id': 207210,\n",
       " 'merchant_address_id': 207210,\n",
       " 'purchase_Is_month_end': 3,\n",
       " 'purchase_Is_month_start': 3,\n",
       " 'purchase_Is_quarter_end': 3,\n",
       " 'purchase_Is_quarter_start': 3,\n",
       " 'purchase_Is_year_end': 3,\n",
       " 'purchase_Is_year_start': 3}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_c = {n: len(c.cat.categories)+1 for n,c in df_valid[cat_flds].items()}\n",
    "emb_c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_szs = [(c, min(30, (c+1)//2)) for _,c in emb_c.items()]\n",
    "metrics=[rmse]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_range=(np.min(y)*1,np.max(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(3, 2),\n",
       " (4, 2),\n",
       " (328, 30),\n",
       " (3, 2),\n",
       " (13, 7),\n",
       " (53, 27),\n",
       " (32, 16),\n",
       " (8, 4),\n",
       " (366, 30),\n",
       " (25, 13),\n",
       " (61, 30),\n",
       " (61, 30),\n",
       " (3, 2),\n",
       " (3, 2),\n",
       " (101, 30),\n",
       " (101, 30),\n",
       " (101, 30),\n",
       " (101, 30),\n",
       " (101, 30),\n",
       " (101, 30),\n",
       " (29, 15),\n",
       " (17, 9),\n",
       " (109392, 30),\n",
       " (42, 21),\n",
       " (3, 2),\n",
       " (6, 3),\n",
       " (6, 3),\n",
       " (4, 2),\n",
       " (7, 4),\n",
       " (13, 7),\n",
       " (3, 2),\n",
       " (272, 30),\n",
       " (26, 13),\n",
       " (6, 3),\n",
       " (644, 30),\n",
       " (647, 30),\n",
       " (488, 30),\n",
       " (505, 30),\n",
       " (233, 30),\n",
       " (234, 30),\n",
       " (6, 3),\n",
       " (4, 2),\n",
       " (3, 2),\n",
       " (6, 3),\n",
       " (8, 4),\n",
       " (3, 2),\n",
       " (8, 4),\n",
       " (10, 5),\n",
       " (2005, 30),\n",
       " (1955, 30),\n",
       " (1963, 30),\n",
       " (1991, 30),\n",
       " (233, 30),\n",
       " (5, 3),\n",
       " (36248, 30),\n",
       " (207210, 30),\n",
       " (207210, 30),\n",
       " (3, 2),\n",
       " (3, 2),\n",
       " (3, 2),\n",
       " (3, 2),\n",
       " (3, 2),\n",
       " (3, 2)]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_szs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(63, 27, 91)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cat_flds), len(cont_flds), len(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3433782, 91)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorized_flag</th>\n",
       "      <th>card_id</th>\n",
       "      <th>installments</th>\n",
       "      <th>category_3</th>\n",
       "      <th>merchant_category_id</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th>purchase_amount</th>\n",
       "      <th>purchase_date</th>\n",
       "      <th>purchase_Year</th>\n",
       "      <th>...</th>\n",
       "      <th>hist_last_buy</th>\n",
       "      <th>installments_ratio</th>\n",
       "      <th>month_diff_ratio</th>\n",
       "      <th>new_first_buy</th>\n",
       "      <th>new_last_buy</th>\n",
       "      <th>observed_elapsed_time</th>\n",
       "      <th>outliers</th>\n",
       "      <th>purchase_amount_sum_ratio</th>\n",
       "      <th>quarter</th>\n",
       "      <th>installments_na</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>643378</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>0.295405</td>\n",
       "      <td>2</td>\n",
       "      <td>269</td>\n",
       "      <td>164944</td>\n",
       "      <td>1.091061</td>\n",
       "      <td>-0.000064</td>\n",
       "      <td>0.767393</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643379</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>1.999314</td>\n",
       "      <td>3</td>\n",
       "      <td>321</td>\n",
       "      <td>526</td>\n",
       "      <td>1.091061</td>\n",
       "      <td>-0.002828</td>\n",
       "      <td>0.801695</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643380</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>0.295405</td>\n",
       "      <td>2</td>\n",
       "      <td>263</td>\n",
       "      <td>183722</td>\n",
       "      <td>1.091061</td>\n",
       "      <td>-0.003234</td>\n",
       "      <td>0.802041</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643381</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>0.295405</td>\n",
       "      <td>2</td>\n",
       "      <td>86</td>\n",
       "      <td>83008</td>\n",
       "      <td>1.091061</td>\n",
       "      <td>-0.003209</td>\n",
       "      <td>0.807492</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643382</th>\n",
       "      <td>1</td>\n",
       "      <td>6769</td>\n",
       "      <td>0.295405</td>\n",
       "      <td>2</td>\n",
       "      <td>282</td>\n",
       "      <td>55457</td>\n",
       "      <td>1.091061</td>\n",
       "      <td>-0.003539</td>\n",
       "      <td>0.811787</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643383</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>0.295405</td>\n",
       "      <td>2</td>\n",
       "      <td>282</td>\n",
       "      <td>55457</td>\n",
       "      <td>1.091061</td>\n",
       "      <td>-0.003539</td>\n",
       "      <td>0.811813</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643384</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>0.295405</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>169141</td>\n",
       "      <td>1.091061</td>\n",
       "      <td>-0.003311</td>\n",
       "      <td>0.833055</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643385</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>0.295405</td>\n",
       "      <td>2</td>\n",
       "      <td>269</td>\n",
       "      <td>131376</td>\n",
       "      <td>1.091061</td>\n",
       "      <td>-0.001745</td>\n",
       "      <td>0.833306</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643386</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>-0.556550</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>115522</td>\n",
       "      <td>1.091061</td>\n",
       "      <td>-0.003281</td>\n",
       "      <td>0.842458</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>11.518360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643387</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>0.295405</td>\n",
       "      <td>2</td>\n",
       "      <td>160</td>\n",
       "      <td>178958</td>\n",
       "      <td>1.091061</td>\n",
       "      <td>-0.003539</td>\n",
       "      <td>0.848663</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643388</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>1.147359</td>\n",
       "      <td>3</td>\n",
       "      <td>196</td>\n",
       "      <td>171153</td>\n",
       "      <td>1.091061</td>\n",
       "      <td>-0.003135</td>\n",
       "      <td>0.851916</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643389</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>-0.556550</td>\n",
       "      <td>1</td>\n",
       "      <td>190</td>\n",
       "      <td>184010</td>\n",
       "      <td>1.357698</td>\n",
       "      <td>-0.002427</td>\n",
       "      <td>0.915228</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>11.518360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643390</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>1.147359</td>\n",
       "      <td>3</td>\n",
       "      <td>28</td>\n",
       "      <td>198513</td>\n",
       "      <td>1.357698</td>\n",
       "      <td>-0.002533</td>\n",
       "      <td>0.916183</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643391</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>0.295405</td>\n",
       "      <td>2</td>\n",
       "      <td>216</td>\n",
       "      <td>65693</td>\n",
       "      <td>1.357698</td>\n",
       "      <td>-0.003530</td>\n",
       "      <td>1.089687</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643392</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>0.295405</td>\n",
       "      <td>2</td>\n",
       "      <td>134</td>\n",
       "      <td>101128</td>\n",
       "      <td>1.357698</td>\n",
       "      <td>-0.003227</td>\n",
       "      <td>1.096555</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643393</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>1.147359</td>\n",
       "      <td>3</td>\n",
       "      <td>190</td>\n",
       "      <td>137752</td>\n",
       "      <td>1.357698</td>\n",
       "      <td>-0.002322</td>\n",
       "      <td>1.114606</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643394</th>\n",
       "      <td>2</td>\n",
       "      <td>6769</td>\n",
       "      <td>0.295405</td>\n",
       "      <td>2</td>\n",
       "      <td>102</td>\n",
       "      <td>180318</td>\n",
       "      <td>1.624336</td>\n",
       "      <td>-0.002657</td>\n",
       "      <td>1.150165</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>0.026446</td>\n",
       "      <td>-0.749105</td>\n",
       "      <td>168</td>\n",
       "      <td>190</td>\n",
       "      <td>23</td>\n",
       "      <td>-0.102886</td>\n",
       "      <td>-0.217039</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.086818</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17 rows × 91 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        authorized_flag  card_id  installments  category_3  \\\n",
       "643378                2     6769      0.295405           2   \n",
       "643379                2     6769      1.999314           3   \n",
       "643380                2     6769      0.295405           2   \n",
       "643381                2     6769      0.295405           2   \n",
       "643382                1     6769      0.295405           2   \n",
       "643383                2     6769      0.295405           2   \n",
       "643384                2     6769      0.295405           2   \n",
       "643385                2     6769      0.295405           2   \n",
       "643386                2     6769     -0.556550           1   \n",
       "643387                2     6769      0.295405           2   \n",
       "643388                2     6769      1.147359           3   \n",
       "643389                2     6769     -0.556550           1   \n",
       "643390                2     6769      1.147359           3   \n",
       "643391                2     6769      0.295405           2   \n",
       "643392                2     6769      0.295405           2   \n",
       "643393                2     6769      1.147359           3   \n",
       "643394                2     6769      0.295405           2   \n",
       "\n",
       "        merchant_category_id  merchant_id  month_lag  purchase_amount  \\\n",
       "643378                   269       164944   1.091061        -0.000064   \n",
       "643379                   321          526   1.091061        -0.002828   \n",
       "643380                   263       183722   1.091061        -0.003234   \n",
       "643381                    86        83008   1.091061        -0.003209   \n",
       "643382                   282        55457   1.091061        -0.003539   \n",
       "643383                   282        55457   1.091061        -0.003539   \n",
       "643384                    28       169141   1.091061        -0.003311   \n",
       "643385                   269       131376   1.091061        -0.001745   \n",
       "643386                     8       115522   1.091061        -0.003281   \n",
       "643387                   160       178958   1.091061        -0.003539   \n",
       "643388                   196       171153   1.091061        -0.003135   \n",
       "643389                   190       184010   1.357698        -0.002427   \n",
       "643390                    28       198513   1.357698        -0.002533   \n",
       "643391                   216        65693   1.357698        -0.003530   \n",
       "643392                   134       101128   1.357698        -0.003227   \n",
       "643393                   190       137752   1.357698        -0.002322   \n",
       "643394                   102       180318   1.624336        -0.002657   \n",
       "\n",
       "        purchase_date  purchase_Year  ...  hist_last_buy  installments_ratio  \\\n",
       "643378       0.767393              1  ...            195            0.026446   \n",
       "643379       0.801695              1  ...            195            0.026446   \n",
       "643380       0.802041              1  ...            195            0.026446   \n",
       "643381       0.807492              1  ...            195            0.026446   \n",
       "643382       0.811787              1  ...            195            0.026446   \n",
       "643383       0.811813              1  ...            195            0.026446   \n",
       "643384       0.833055              1  ...            195            0.026446   \n",
       "643385       0.833306              1  ...            195            0.026446   \n",
       "643386       0.842458              1  ...            195            0.026446   \n",
       "643387       0.848663              1  ...            195            0.026446   \n",
       "643388       0.851916              1  ...            195            0.026446   \n",
       "643389       0.915228              2  ...            195            0.026446   \n",
       "643390       0.916183              2  ...            195            0.026446   \n",
       "643391       1.089687              2  ...            195            0.026446   \n",
       "643392       1.096555              2  ...            195            0.026446   \n",
       "643393       1.114606              2  ...            195            0.026446   \n",
       "643394       1.150165              2  ...            195            0.026446   \n",
       "\n",
       "        month_diff_ratio  new_first_buy  new_last_buy  observed_elapsed_time  \\\n",
       "643378         -0.749105            168           190                     23   \n",
       "643379         -0.749105            168           190                     23   \n",
       "643380         -0.749105            168           190                     23   \n",
       "643381         -0.749105            168           190                     23   \n",
       "643382         -0.749105            168           190                     23   \n",
       "643383         -0.749105            168           190                     23   \n",
       "643384         -0.749105            168           190                     23   \n",
       "643385         -0.749105            168           190                     23   \n",
       "643386         -0.749105            168           190                     23   \n",
       "643387         -0.749105            168           190                     23   \n",
       "643388         -0.749105            168           190                     23   \n",
       "643389         -0.749105            168           190                     23   \n",
       "643390         -0.749105            168           190                     23   \n",
       "643391         -0.749105            168           190                     23   \n",
       "643392         -0.749105            168           190                     23   \n",
       "643393         -0.749105            168           190                     23   \n",
       "643394         -0.749105            168           190                     23   \n",
       "\n",
       "        outliers  purchase_amount_sum_ratio  quarter  installments_na  \n",
       "643378 -0.102886                  -0.217039        2        -0.086818  \n",
       "643379 -0.102886                  -0.217039        2        -0.086818  \n",
       "643380 -0.102886                  -0.217039        2        -0.086818  \n",
       "643381 -0.102886                  -0.217039        2        -0.086818  \n",
       "643382 -0.102886                  -0.217039        2        -0.086818  \n",
       "643383 -0.102886                  -0.217039        2        -0.086818  \n",
       "643384 -0.102886                  -0.217039        2        -0.086818  \n",
       "643385 -0.102886                  -0.217039        2        -0.086818  \n",
       "643386 -0.102886                  -0.217039        2        11.518360  \n",
       "643387 -0.102886                  -0.217039        2        -0.086818  \n",
       "643388 -0.102886                  -0.217039        2        -0.086818  \n",
       "643389 -0.102886                  -0.217039        2        11.518360  \n",
       "643390 -0.102886                  -0.217039        2        -0.086818  \n",
       "643391 -0.102886                  -0.217039        2        -0.086818  \n",
       "643392 -0.102886                  -0.217039        2        -0.086818  \n",
       "643393 -0.102886                  -0.217039        2        -0.086818  \n",
       "643394 -0.102886                  -0.217039        2        -0.086818  \n",
       "\n",
       "[17 rows x 91 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[643378:643395]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idx2 = list(range(0, 643395))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "643395"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(val_idx2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1874"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(643395, (3433782, 91), (3433782,))"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(val_idx2), df.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "91"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.unique(df.columns.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "md = ColumnarModelData.from_data_frame('./', val_idx2, df, y.astype(np.float32), cat_flds=cat_flds, bs=512, test_df=df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.85 s, sys: 306 ms, total: 2.15 s\n",
      "Wall time: 2.52 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "m = md.get_learner(emb_szs, len(df.columns)-len(cat_flds),\n",
    "                   0.4, 1, [1000, 500], [0.1, 0.01], y_range=y_range)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebb1f4ebfa4442808d85da60fb8ca07a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 2904/5450 [00:50<00:43, 58.69it/s, loss=1.68] "
     ]
    }
   ],
   "source": [
    "m.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEOCAYAAACEiBAqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmcXFWZ//HPU1tX70u6s+9AkB0hIAoKiCAuIyiIzCiDykxcxhF1nJ84zm9GRn8jOu644jJmVETHBRCQRWR1CSQBhBCQQELI3km603t1V9fz++PeJJ3QSyXp29VV9X2/XvdV9966y1M3nXrqnHPvOebuiIhI+YoVOgARESksJQIRkTKnRCAiUuaUCEREypwSgYhImVMiEBEpc0oEIiJlTolARKTMKRGIiJQ5JQIRkTKXKHQA+Whubvb58+cXOgwRkaKyYsWK7e7eMtZ2RZEI5s+fz/LlywsdhohIUTGz5/PZTlVDIiJlTolARKTMKRGIiJQ5JQIRkTKnRCAiUuaUCEREylykt4+a2TqgExgEsu6+2MyagJ8C84F1wCXu3hbF+Veub6M/m+O0hVOiOLyISEmYiBLB2e5+orsvDpevAu529yOAu8PlSFx79zP8522rozq8iEhJKETV0AXA0nB+KXBhVCdKxmP0Z3NRHV5EpCREnQgcuNPMVpjZknDdNHffDBC+To3q5MlEjIFBJQIRkdFE3cXE6e6+ycymAneZ2VP57hgmjiUAc+fOPaiTp+IxBgb9oPYVESkXkZYI3H1T+LoN+BVwKrDVzGYAhK/bRtj3Ondf7O6LW1rG7DNpWEEiUIlARGQ0kSUCM6s2s9rd88B5wBPAzcDl4WaXAzdFFUMyYUoEIiJjiLJqaBrwKzPbfZ7r3f12M3sY+JmZXQGsB94aVQBqLBYRGVtkicDdnwNOGGb9DuCcqM47VCoeo18lAhGRUZX0k8VJNRaLiIyp5BPBYM4ZzCkZiIiMpLQTQcIA1GAsIjKKkk4EqXjw8ZQIRERGVtKJILknEahqSERkJGWRCHQLqYjIyEo6EaQSSgQiImMp6URQESaCTHawwJGIiExeZZIIVCIQERlJSSeClBKBiMiYSjoRVCTigKqGRERGU9qJIKnGYhGRsZR0Itj9QJmqhkRERlbSiSCdVCIQERlLSSeC3W0EqhoSERlZSSeClJ4jEBEZU0kngj3PEQyoRCAiMpISTwRh1ZB6HxURGVFJJ4KUSgQiImMq6UQQjxmJmKmNQERkFCWdCCBoJ9BdQyIiIyv5RJBKxPQcgYjIKEo+EVQk4qoaEhEZRekngqRKBCIioyn5RJCKq41ARGQ0JZ8IVCIQERld6ScCtRGIiIyq5BOBqoZEREZX8olAVUMiIqMr/USQiKmLCRGRUZR8Ikgl4up0TkRkFJEnAjOLm9kjZnZLuLzAzJaZ2TNm9lMzS0V5/qBEoMZiEZGRTESJ4Epg9ZDlzwJfcvcjgDbgiihPXqEuJkRERhVpIjCz2cAbgO+Gywa8Gvh5uMlS4MIoY6hIxHXXkIjIKKIuEXwZ+D/A7m/iKUC7u2fD5Q3ArCgDUKdzIiKjiywRmNkbgW3uvmLo6mE29RH2X2Jmy81seWtr60HHUZGI0T+YI5cb9jQiImUvyhLB6cCbzGwdcANBldCXgQYzS4TbzAY2Dbezu1/n7ovdfXFLS8tBB1GRDD6i7hwSERleZInA3T/u7rPdfT5wKfA7d387cA9wcbjZ5cBNUcUAe8ct1rMEIiLDK8RzBB8DPmJmawjaDL4X5cmqUkEi6BnIjrGliEh5Soy9yaFz93uBe8P554BTJ+K8sDcRdGf0LIGIyHBK/sni6lSQ63r6VSIQERlOySeCqgqVCERERlP6iSAsEfSqjUBEZFglnwgqk0GJoLdfdw2JiAynfBKBOp4TERlW6SeC1O4SgaqGRESGUz6JQCUCEZFhlX4iUBuBiMioSj4RxGNGKhHTk8UiIiMo+UQAQamgr19VQyIiwymbRKA2AhGR4ZVFIqhKxelRiUBEZFhlkQjSyTh9KhGIiAyrLBJBZUpVQyIiIymLRKCqIRGRkZVFIkgn4/QqEYiIDGvMRGBm1WYWC+cXmdmbzCwZfWjjp1JtBCIiI8qnRHA/kDazWcDdwLuAH0QZ1HhT1ZCIyMjySQTm7j3AW4Br3f3NwNHRhjW+KlOqGhIRGUleicDMXg68Hbg1XDchYx2Pl6pUnO7+LO5e6FBERCadfBLBh4CPA79y91VmthC4J9qwxldVKkHOIZNVx3MiIvsb85e9u98H3AcQNhpvd/cPRh3YeKoKu6Lu6R8kHfZGKiIigXzuGrrezOrMrBp4EnjazP45+tDGT3U4bnGPBqcREXmRfKqGjnb3DuBC4DZgLnBZpFGNs72jlKnBWERkf/kkgmT43MCFwE3uPgAUVatrdUWQCLqVCEREXiSfRPBtYB1QDdxvZvOAjiiDGm9VYdVQd0ZVQyIi+8unsfirwFeHrHrezM6OLqTxV1MRfMwuJQIRkRfJp7G43sy+aGbLw+kLBKWDorE7EahEICLyYvlUDX0f6AQuCacO4L+jDGq8VSsRiIiMKJ8nhA9z94uGLF9tZo9GFVAU9lYNqbFYRGR/+ZQIes3sjN0LZnY60DvWTmaWNrOHzOwxM1tlZleH6xeY2TIze8bMfmpmqYMPPz/pZIyYqUQgIjKcfBLB+4Cvm9k6M3se+Brw3jz2ywCvdvcTgBOB883sNOCzwJfc/QigDbji4ELPn5lRU5FQY7GIyDDyuWvoUeAEM6sLl/O6ddSDHt66wsVkODnwauBvwvVLgU8C3zygqA9CbTrJzu7+qE8jIlJ0RkwEZvaREdYD4O5fHOvgZhYHVgCHA18HngXa3X33T/MNwKwDC/ngzG6sZGP7mDVaIiJlZ7QSQe2hHtzdB4ETzawB+BVw1HCbDbevmS0BlgDMnTv3UEOhsSrFc9u7xt5QRKTMjJgI3P3q8TqJu7eb2b3AaUCDmSXCUsFsYNMI+1wHXAewePHiQ+7SojadoLNPbQQiIvuLbPB6M2sJSwKYWSXwGmA1wVgGF4ebXQ7cFFUMQ9UoEYiIDCvKkcZmAEvDdoIY8DN3v8XMngRuMLNPA48A34swhj1q00m6MllyOScWs4k4pYhIUYgsEbj7n4GXDrP+OeDUqM47krp0+FBZf5a6dHKiTy8iMmmNmQjMrAK4CJg/dHt3/4/owhp/tWEi6OxTIhARGSqfNoKbgAuALNA9ZCoqteGXf0fvQIEjERGZXPKpGprt7udHHknEGquCnizaevRQmYjIUPmUCP5gZsdFHknEmqrDRNCtEoGIyFD5lAjOAN5pZmsJ+g8ygh4kjo80snHWXBMkgm2dfQWORERkcsknEbwu8igmQFN1iopEjM27lAhERIYas2rI3Z8HGoC/CqeGcF1RMTNm1KfZpP6GRET2kc9QlVcCPwamhtOPzOwfow4sCjPqK1UiEBHZTz5VQ1cAL3P3bgAz+yzwR+DaKAOLwoyGNH98dkehwxARmVTyuWvIgKFjPA6G64rOzPpKtnb0kR3MFToUEZFJI58SwX8Dy8zsV+HyhUxQ/0DjbVpdBTmHnT39TK1NFzocEZFJIZ8Ryr4YdiF9BkFJ4F3u/kjUgUWhJfzy39aRUSIQEQmNNkJZnbt3mFkTsC6cdr/X5O47ow9vfDVUBd1MPL2lk2Nn1Rc4GhGRyWG0NoLrw9cVwPIh0+7lopOMBx/383c+XeBIREQmj9FGKHtj+Lpg4sKJ1snzGgF4+cIpBY5ERGTyyOc5grvzWVcsjp9dz45udTwnIrLbaG0EaaAKaDazRvbeMloHzJyA2CLRNzDIfX9ppT+bI5WIbKROEZGiMdpdQ+8BPkTwpb+CvYmgA/h6xHFFJh4LvvzX7+zm8Km1BY5GRKTwRvxJ7O5fCdsHPuruC919QTid4O5fm8AYx9X/e/OxALywU30OiYhAfs8RXGtmxwJHA+kh6/8nysCi0lJTAcD2rkyBIxERmRzyGbP434GzCBLBbQTdUj8IFGUi2D1AzbZOJQIREcivr6GLgXOALe7+LuAEoCLSqCJUlYoD8F936FkCERHILxH0unsOyJpZHbANWBhtWNEx29tf3oA6nxMRySsRLDezBuA7BHcPrQQeijSqiH32omAI5i0am0BEJK/G4veHs98ys9uBOnf/c7RhRWtOYxUAL+zsYU5TVYGjEREprNEeKDtptPfcfWU0IUVv9u5E0NZT4EhERApvtBLBF8LXNLAYeIzgobLjgWUE3VIXpRkNaWKmZwlERGD0B8rOdvezgeeBk9x9sbufDLwUWDNRAUYhGY8xu7GK1Zs7Ch2KiEjB5dNY/BJ3f3z3grs/AZwYXUgTY0FzNXc/ta3QYYiIFFw+Q1WuNrPvAj8CHHgHsDrSqCZAZTJ4nqC1M0NLbdE+FiEicsjyKRG8C1gFXEnQCd2T4bpRmdkcM7vHzFab2SozuzJc32Rmd5nZM+Fr46F8gIN18cmzAVi7vbsQpxcRmTTGTATu3ufuX3L3N4fTl9w9nxvws8A/uftRwGnAP5jZ0cBVwN3ufgRwd7g84Y6YVgPAuh1KBCJS3kZMBGb2s/D1cTP78/7TWAd29827bzF1906C6qRZwAXA0nCzpcCFh/ohDsashkoAvv/g2kKcXkRk0hitjeDK8PWNh3oSM5tPcLfRMmCau2+GIFmY2dRDPf7BSITjFz+1pZNtnX1MrU2PsYeISGka7fbR3V/Wzw835XsCM6sBfgF8yN3zvl/TzJaY2XIzW97a2prvbgfk25edDMD7flS0z8aJiByy0aqGOs2sY5ip08zy+kI3syRBEvixu/8yXL3VzGaE788g6MTuRdz9uvDZhcUtLS0H9qny9NpjppNKxHhi465Iji8iUgxGKxHUunvdMFOtu9eNdWALuvn8HrDa3b845K2bgcvD+cuBmw7lAxyqj53/EjLZHNs61QGdiJSnvEdvN7OpZjZ395THLqcDlwGvNrNHw+n1wDXAuWb2DHBuuFwwR00Pxi3+/oPrChmGiEjB5DNC2ZsI+h2aSVCNM4/gDqBjRtvP3R9k74D3+zvnwMKMzglzGgBYtnZHgSMRESmMfEoEnyJ4DuAv4WD25wC/jzSqCVRdkeA1R03lkfXt7NA4xiJShvJJBAPuvgOImVnM3e+hBPoaGqq+MhjH+ORP/7bAkYiITLx8EkF7eAvo/cCPzewrBE8Nl4yrLwhquRKxkWqyRERKVz6J4AKgB/gwcDvwLPBXUQY10WoqEvzty+eRzbm6phaRspNPIlgCzHT3rLsvdfevhlVFJeX1x80A4HVfeYDbHt9c4GhERCZOPomgDrjDzB4ws38ws2lRB1UIpy2csmf+/T9eSVt3fwGjERGZOPn0Pnq1ux8D/APBLaT3mVlJtqp+6oK9d8R++taiH3JBRCQveT9QRvAMwRZgB1CQjuKidtnL5/PwJ14DwC9WbuDvlj5MdjBX4KhERKI1ZiIws/eZ2b0EYwc0A3/v7sdHHVihDB2t7Lert3H4J35TwGhERKKXT4lgHkHPoce4+7+7+5NRB1VoD/3LOZw0t2HP8nOtXQWMRkQkWvm0EVzl7o9ORDCTxdS6NL98/+nc89GzAHj1F+7jzlVbChuUiEhEDqSNoOwsaK7eM7/khyv4l189XsBoRESioUQwhhuWnLZn/vpl6wsYiYhINJQIxnDawimsu+YNnHF4MwB3r97K/Ktu5ZH1bQBksoO4eyFDFBE5JEoEefrIeYsAuGLpcgDe/I0/sKm9lyP/9XZ1ViciRU2JIE8nzm540bpXXPM7AHZ29/PM1s6JDklEZFxYMVRrLF682JcvX17oMPZwdxZ8/LYXrZ9Rn+Y7f7uYY2fVFyAqEZF9mdkKd1881nYqERwEM+NPHz+H9555GCv/77l71m/e1ccbr32QXG7yJ1cRkd2UCA7S9Po0V73uJTRVp1h3zRv47EXH7Xnvoz9/rICRiYgcGCWCcfK2U+by1KfOB+CXKzcyoD6KRKRIKBGMo3QyzjfffhIAR6iPIhEpEkoE4+w1R+8druF3T20tYCQiIvlRIhhnyXiMWz94BgDv/sFy7n16W4EjEhEZnRJBBI6ZWc/Sd58KwDv/+2Ey2cECRyQiMjIlgoicuaiF95y5EIAj//V2dWUtIpOWEkGEPvbal+yZf/UX7uMOdWUtIpOQEkGEYjHjh1ecumf5PT9cwQ//uK5g8YiIDEddTEyQu1dv3dNhHcDM+jR3feRMqisSBYxKREqZupiYZM45ahrfesdJe5Y37erj0uv+VMCIREQCSgQT6PxjZ/D4J8/ji5ecAMDjG3exMhzXQESkUJQIJlhtOslbTprNqqtfS3NNii/c+XShQxKRMhdZIjCz75vZNjN7Ysi6JjO7y8yeCV8bozr/ZFddkeBdpy/g92t28JZv/J5dvQOFDklEylSUJYIfAOfvt+4q4G53PwK4O1wuW5eeMofZjZWsXN/OCVffSU9/ttAhiUgZiiwRuPv9wM79Vl8ALA3nlwIXRnX+YjClpoIHP/Zq3nDcDAD+5jvL6OhTyUBEJtZEtxFMc/fNAOHr1Ak+/6T09befxAUnzuTRF9r55M2rCh2OiJSZSdtYbGZLzGy5mS1vbW0tdDiR+/LbTuSsI1u48ZGNPLWlo9DhiEgZmehEsNXMZgCEryN2zenu17n7Yndf3NLSMmEBFoqZ8aVLTqQiEeffblxFMTzoJyKlYaITwc3A5eH85cBNE3z+Sa2xOsU/nbeIh9bt5CcPvVDocESkTER5++hPgD8CR5rZBjO7ArgGONfMngHODZdliHe+Yj4Lmqv5t5ue4IFnSr9KTEQKT30NTUIv7OzhlZ+7B4CaigR3feRVzKivLHBUIlJs1NdQEZvTVMVvrnwliZjRlcny7h8sp1O3lYpIRJQIJqmjZtSx5j9fz9f+5qU8s7WTS6/7Ew8+s52+AY12JiLjS4lgknvj8TP5zuWLeX5HD+/43jLO+9L97OjKFDosESkhSgRF4Owjp/KbK1/JxSfPZv3OHk7+9G/5zeObCx2WiJQIJYIiMaepis+/9QSWvjsY8ewDP3lEyUBExoUSQZE5c1ELj/7buSxsruZ9P17J525/ikxW7QYicvCUCIpQQ1WKH//9yzh1QRPfuPdZTv7Ub1n23I5ChyUiRUqJoEhNrU3z0yWn8fm3noABb7vuT7z3hyvY2d1f6NBEpMgoERQxM+Pik2dzzz+fxYUnzuT2VVt43Vfu5/drthc6NBEpIkoEJaC5poIvX/pSbv3gGdRUJHjH95bxmd+s1jMHIpIXdTFRYnr7B/nUrU9y/bL11FYkOGxqDVeecwRnLmohFrNChyciEyjfLiaUCErUQ2t3svQP67g1vMX0lPmNXHPR8RzWUlPgyERkoigRCACdfQP8+rHNfO6Op+jOZFnyqoVcespc5jRVFTo0EYmYEoHsY1tHH9f85il++chGAE6e18j5x0znmFl1nDq/iURczUUipUaJQIb1xMZd/OHZ7dzw8As819oNQH1lkiOn1XLSvEYuWTybhao+EikJSgQyKnentSvDQ2t3ct/TrTy9tZPVmzsYGHSOnVXHaQumcOaRLZwyv4l0Ml7ocEXkICgRyAFr7cxww0PreXDNdlaub2Ng0KlKxVnYUs1R0+t4xeFTeMVhzTTXVBDXHUgik54SgRySjr4Bfv/Mdv7w7A7W7ejmiY27aOsJBsdJJ2NMrU3TVJ2ipiJB/2COuU1V1KWTvGxhE688opmqVKLAn0BElAhkXOVyzqpNHTy0bieb23tp7cqws7ufnd399GdzbNnVR2cmC0A8ZkyvSzO9Pk1jVYojp9ewoLmGmoo4ZsbU2goWTaulukLJQiRK+SYC/U+UvMRixnGz6zludv2I23Rlsjyyvo1lz+1kQ1sPT27uoLUzwz1Pb2Mwt+8PDjM4dmZwvBPnNHBYSzUzGyqZVpvWg28iE0wlAolcfzbHpvZeevoHybmzoa2H1Zs7+eOzO1i9uWNPSQIglYgxu7GS+VOqqUsnmNlQyZHTa5k3pZpF02pU5SRyAFQ1JEVhMOes39nD2u1dbGzvY8POHtbv7OG51m66Mlm2dvSRHVKaaKxKMrepivnN1SxsrmF+cxWzGiqZ1VjJ1Nq0GrFFhlDVkBSFeMxY0FzNgubqYd/vGxjk2dYunm3tZv2ObtZu7+GFth5WPN/GzY9tYujvmETMmNlQyezGYDpqRh0nzGlg3fZuFk2r5agZdUoUIsNQIpBJLZ2Mc8zMeo6Z+eK2ie5Mlo3tvcHUFrxuaOtlY1sPv3uqlZ8t37DP9lWpOMfPrmfxvCaOnlnHomm1zJ9SpaeqpewpEUjRqq5IsGhaLYum1Q77/sb2XlY838am9l6m16V5ZH0bK9a38c37nt3TeJ2Kxzh8ag0nzm3ghNn1vOKwZmY3VmKmkoOUD7URSNnpGxhkzbYunt7SyV+2drJqUwePvdC+p9G6qTrFYS3V9PQPMr0uzcKWauY2VXHY1BpmN1Qxp0mJQoqD2ghERpBOxjl2Vj3Hztpb3eTuPLOti2Vrd/Lo+nbW7+ympbaCje29PLBmO/3Z3J5ta9MJFjZXM6O+klQiRmUyzpymSk6c08jClmqm1anRWoqLEoEIwbCfu6uZLjtt3j7v5XLOlo4+1m7v5vkdPTy5eRfP7+hhTWsX7T0DZHM52sOnrgEqEjHmNAV3Mx3WUsO8KcFdTgumVDOrsVJJQiYdJQKRMcTCu5FmNlRy+uHDb9Pe08+qTR2s3d7Nuu3dbGjr5fmdPTy8bic9/XuHDE3GjblNVcyor6Qzk6UunaCxKkVdZYKWmjQLWqpprk4xta6C+soU9ZVJUol9G7N3dvcDwVgTXZks7jB3StDFh8jBUCIQGQcNVSlOP7yZ0w9v3mf97l5e17Z2sy68/XXd9m62dvZRl06wq3eAF3b2sKt3YE9fTvurTsWJxYzMQA7HGRgcvl1vYUs1i6bWctjUarr6smza1Rc8Y9FQyZSa1J7OAisSMabXp5lam35RkpHyVJBEYGbnA18B4sB33f2aQsQhErWgb6XgS/dlC6eMum3fwCDrdnSzo6ufHd397Orpp71ngPbeATLZQWrDX/zJeIyGyiS16QS16eC/8JptXTy2YRd/2dbJXau37rkrqqYiQdeQJ7f3V1+ZxAwMyOacunSSaXUV1FcmaaquYEpNiinVKeIxo7d/kHQyTmUqzvS6NI3VSSqTCRqqkuTc2daZIZdzUokYnX1Z4jGjvjK5Z6pKBd2Zu5NXNyLuTmcmS8yMLbv6qKtMUJdO7tMt+oPPbOe2JzZzxNQajpxey8LmGnZ0Z1i5vp3Wjj6m1afpG8gxr6mKTDbHs61dNFWn6M5kyeY8aOtpqGQw57T39LOrd4BkPEZdZRJ3D6+501iV4t6nW+npz5LJ5tjQ1gvAguZqqisSNFYl2dU7QMyM1Zs7SCfjHD61hlmNlXRnsmxq7yUVj5HNOV2ZLL0Dg7R2ZNjWmaGxOsWxM+s4emYdleH13dDWS206wdTaCuY2VUeesCc8EZhZHPg6cC6wAXjYzG529ycnOhaRySSdjPOS6XWHfJz+bI6N7b3MaqgkGTc6M1m2d2bY3tXPYM7JZAfZ2tHHll0ZdnZn6OzLUpGM05/Nkc3l2N6VobUrw1+2dtHaldmnofxQJGJGzIz+wRw1FQnq0gnq9iS0IMnl3OntH6S9Z4A1rV0v6qNq93GqKxL0DQySGafYDkZ9ZfDlv7+YBQ9KjlRy2606FaeltoK127v59WObRtzurg+/iiNGuEV6vBSiRHAqsMbdnwMwsxuACwAlApFxkErE9nlSuy6dpC6dZGHLgR/LPfgF250ZZEpNir6BQbozg2zp6KOjd4DuTJZdvQMMevCruaYiQSabYzDn1IZVX7unjt4BBnNORTJOV1+Wjr5gXUffAFs7+oiFt+SmkzGm1ac5eX4j0+vSADTXVDDovuecnX1ZMtlB5k2p5vxjp1ObTvDU5k7WbOsiHgt6uD1yei09/YPUpZNs786wvTPDiXMbGMw5nX1ZZtSneX5HD+t2dJOMx4iZMaepck8JJBmP0ZXJMphzdnRnWNhcw7Gz6ujtD0pnlak4u8LSWkdvlqpUnHjMaKmpAGBLRx+b2nuJx4z5U6pxgi5Vqivi4fXZ26bT1t3P6s0dtPUMkHNnVmMlfQODtHZmmNVYefB/DHma8OcIzOxi4Hx3/7tw+TLgZe7+gZH20XMEIiIHLt/nCArRUjRc5eCLspGZLTGz5Wa2vLW1dQLCEhEpT4VIBBuAOUOWZwMvqiBz9+vcfbG7L25pOYgyrYiI5KUQieBh4AgzW2BmKeBS4OYCxCEiIhSgsdjds2b2AeAOgttHv+/uqyY6DhERCRTkOQJ3vw24rRDnFhGRfemxQhGRMqdEICJS5pQIRETKXFEMTGNmrcDz+62uB3aNsa4Z2B5haMMZLq6oj5HP9mNtM9L7+VznkdYV4/WP4tqPtd2BvjdZr385/O2PtH6yfvfMc/ex779396KcgOvGWgcsnwxxRX2MfLYfa5uR3s/nOo+yruiufxTXfqztDvS9yXr9y+FvP9/rX2x/+8VcNfTrPNdNtPGI4UCPkc/2Y20z0vv5XufJcO3h0OOI4tqPtd2BvjdZr385/O2PtL6or39RVA0dLDNb7nn0syHR0PUvLF3/wim2a1/MJYJ8XFfoAMqcrn9h6foXTlFd+5IuEYiIyNhKvUQgIiJjUCIQESlzSgQiImWubBOBmZ1lZg+Y2bfM7KxCx1NuzKzazFaY2RsLHUu5MbOjwr/7n5vZ+wodT7kxswvN7DtmdpOZnVfoeKBIE4GZfd/MtpnZE/utP9/MnjazNWZ21RiHcaALSBMMliN5GKdrD/Ax4GfRRFm6xuP6u/tqd38vcAlQNLc4TgbjdP1vdPe/B94JvC3CcPNWlHcNmdmrCL7E/8fdjw3XxYG/AOcSfLE/DPw1wZgHn9nvEO8Gtrt7zsymAV9097dPVPzFbJyu/fEEj+CnCf4dbpmY6IvfeFx/d99mZm8CrgK+5u7XT1T8xW68rn+43xeAH7v7ygkKf0QFGY/gULn7/WY2f7/VpwJr3P3VdmSbAAAGSklEQVQ5ADO7AbjA3T8DjFb90AZURBFnKRqPa29mZwPVwNFAr5nd5u65SAMvEeP1t+/uNwM3m9mtgBJBnsbp79+Aa4DfTIYkAEWaCEYwC3hhyPIG4GUjbWxmbwFeCzQAX4s2tJJ3QNfe3T8BYGbvJCyZRRpd6TvQv/2zgLcQ/ADSAFGH7oCuP/CPwGuAejM73N2/FWVw+SilRGDDrBux3svdfwn8MrpwysoBXfs9G7j/YPxDKUsH+rd/L3BvVMGUoQO9/l8FvhpdOAeuKBuLR7ABmDNkeTawqUCxlBtd+8LS9S+sor/+pZQIHgaOMLMFZpYCLgVuLnBM5ULXvrB0/Qur6K9/USYCM/sJ8EfgSDPbYGZXuHsW+ABwB7Aa+Jm7rypknKVI176wdP0Lq1Svf1HePioiIuOnKEsEIiIyfpQIRETKnBKBiEiZUyIQESlzSgQiImVOiUBEpMwpEci4M7OuCTjHm/Ls7no8z3mWmb3iIPZ7qZl9N5x/p5lNir6tzGz+/t0pD7NNi5ndPlExSWEoEcikFXbvOyx3v9ndr4ngnKP1v3UWcMCJAPgX4NqDCqjA3L0V2Gxmpxc6FomOEoFEysz+2cweNrM/m9nVQ9bfGI5QtsrMlgxZ32Vm/2Fmy4CXm9k6M7vazFaa2eNm9pJwuz2/rM3sB2b2VTP7g5k9Z2YXh+tjZvaN8By3mNltu9/bL8Z7zew/zew+4Eoz+yszW2Zmj5jZb81sWtj18HuBD5vZo2b2yvDX8i/Cz/fwcF+WZlYLHO/ujw3z3jwzuzu8Nneb2dxw/WFm9qfwmP8xXAnLghHebjWzx8zsCTN7W7j+lPA6PGZmD5lZbfjL/4HwGq4crlRjZnEz+68h/1bvGfL2jYDG6yhl7q5J07hOQFf4eh5wHUHvjDHgFuBV4XtN4Wsl8AQwJVx24JIhx1oH/GM4/37gu+H8OwkGVQH4AfC/4TmOJugbHuBigm6WY8B0grEnLh4m3nuBbwxZbmTvU/d/B3whnP8k8NEh210PnBHOzwVWD3Pss4FfDFkeGvevgcvD+XcDN4bztwB/Hc6/d/f13O+4FwHfGbJcD6SA54BTwnV1BD0MVwHpcN0RwPJwfj7wRDi/BPjXcL4CWA4sCJdnAY8X+u9KU3RTKXVDLZPPeeH0SLhcQ/BFdD/wQTN7c7h+Trh+BzAI/GK/4+zuLnwFQT/6w7nRg3ENnrRg1DmAM4D/DddvMbN7Ron1p0PmZwM/NbMZBF+ua0fY5zXA0cE4IwDUmVmtu3cO2WYG0DrC/i8f8nl+CHxuyPoLw/nrgc8Ps+/jwOfN7LPALe7+gJkdB2x294cB3L0DgtID8DUzO5Hg+i4a5njnAccPKTHVE/ybrAW2ATNH+AxSApQIJEoGfMbdv73PymBglNcAL3f3HjO7l2DYSoA+dx/c7ziZ8HWQkf9mM0Pmbb/XfHQPmb+WYPjSm8NYPznCPjGCz9A7ynF72fvZxpJ3x1/u/hczOxl4PfAZM7uToApnuGN8GNgKnBDG3DfMNkZQ8rpjmPfSBJ9DSpTaCCRKdwDvNrMaADObZWZTCX5ttoVJ4CXAaRGd/0HgorCtYBpBY28+6oGN4fzlQ9Z3ArVDlu8k6HUSgPAX9/5WA4ePcJ4/EHRZDEEd/IPh/J8Iqn4Y8v4+zGwm0OPuPyIoMZwEPAXMNLNTwm1qw8bveoKSQg64jGAs3f3dAbzPzJLhvovCkgQEJYhR7y6S4qZEIJFx9zsJqjb+aGaPAz8n+CK9HUiY2Z+BTxF88UXhFwSDhjwBfBtYBuzKY79PAv9rZg8A24es/zXw5t2NxcAHgcVh4+qTBPX5+3D3pwiGJKzd/71w/3eF1+Ey4Mpw/YeAj5jZQwRVS8PFfBzwkJk9CnwC+LS79wNvA641s8eAuwh+zX8DuNzM/kTwpd49zPG+CzwJrAxvKf02e0tfZwO3DrOPlAh1Qy0lzcxq3L3LzKYADwGnu/uWCY7hw0Cnu383z+2rgF53dzO7lKDh+IJIgxw9nvsJBmNvK1QMEi21EUipu8XMGggafT810Ukg9E3grQew/ckEjbsGtBPcUVQQZtZC0F6iJFDCVCIQESlzaiMQESlzSgQiImVOiUBEpMwpEYiIlDklAhGRMqdEICJS5v4/y8MRkCPRB+gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "m.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=5e-5; wd=1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2d44c5926a84728ae31de2cc8539daf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      0.75214    3.83553    1.869829  \n",
      "    1      0.6297     3.637147   1.816519                      \n",
      "CPU times: user 2min 48s, sys: 29.5 s, total: 3min 18s\n",
      "Wall time: 3min 9s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([3.63715]), 1.8165192963266164]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time m.fit(lr, 1, cycle_len=2, cycle_mult=2, metrics=[rmse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = m.predict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(643395, 1)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3433782, 91)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3433782"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(val_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "l_t = m.predict(is_test=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((3433782, 1), (643395, 1), (3433782, 91))"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l_t.shape, l.shape, df_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "df_valid['target_new'] = l_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>card_id</th>\n",
       "      <th>target</th>\n",
       "      <th>target_new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C_ID_efced389a0</td>\n",
       "      <td>-1.840332</td>\n",
       "      <td>0.658630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C_ID_ecf77791d8</td>\n",
       "      <td>-1.065921</td>\n",
       "      <td>1.004230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>C_ID_a24df943dd</td>\n",
       "      <td>3.130923</td>\n",
       "      <td>0.989231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>C_ID_ce778b86f8</td>\n",
       "      <td>1.162205</td>\n",
       "      <td>0.969341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>C_ID_7ba7751d53</td>\n",
       "      <td>0.679358</td>\n",
       "      <td>0.706181</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            card_id    target  target_new\n",
       "1   C_ID_efced389a0 -1.840332    0.658630\n",
       "5   C_ID_ecf77791d8 -1.065921    1.004230\n",
       "6   C_ID_a24df943dd  3.130923    0.989231\n",
       "15  C_ID_ce778b86f8  1.162205    0.969341\n",
       "18  C_ID_7ba7751d53  0.679358    0.706181"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid[['card_id', 'target', 'target_new']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_haa = df_valid.groupby('card_id').agg({'target': ['median'], 'target_new': ['median']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_haa.columns = ['_'.join(c) for c in df_haa.columns.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.9617159691277544"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse(df_haa.target_median.values, df_haa.target_new_median.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target_median</th>\n",
       "      <th>target_new_median</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C_ID_0005b2f279</th>\n",
       "      <td>2.945304</td>\n",
       "      <td>-1.311934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_0005b5804f</th>\n",
       "      <td>0.685599</td>\n",
       "      <td>-1.000813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_0007a60a33</th>\n",
       "      <td>-1.614517</td>\n",
       "      <td>-1.164612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_000844d8f1</th>\n",
       "      <td>-0.868258</td>\n",
       "      <td>-1.182117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_000a6044e9</th>\n",
       "      <td>-3.774958</td>\n",
       "      <td>-1.102848</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 target_median  target_new_median\n",
       "card_id                                          \n",
       "C_ID_0005b2f279       2.945304          -1.311934\n",
       "C_ID_0005b5804f       0.685599          -1.000813\n",
       "C_ID_0007a60a33      -1.614517          -1.164612\n",
       "C_ID_000844d8f1      -0.868258          -1.182117\n",
       "C_ID_000a6044e9      -3.774958          -1.102848"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_haa.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_vla = df_sorted[:643395]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((643395, 91), (643395, 1))"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_vla.shape, l.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "df_vla['target_pred'] = l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorized_flag</th>\n",
       "      <th>card_id</th>\n",
       "      <th>installments</th>\n",
       "      <th>category_3</th>\n",
       "      <th>merchant_category_id</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th>purchase_amount</th>\n",
       "      <th>purchase_date</th>\n",
       "      <th>purchase_Year</th>\n",
       "      <th>...</th>\n",
       "      <th>installments_ratio</th>\n",
       "      <th>month_diff_ratio</th>\n",
       "      <th>new_first_buy</th>\n",
       "      <th>new_last_buy</th>\n",
       "      <th>observed_elapsed_time</th>\n",
       "      <th>outliers</th>\n",
       "      <th>purchase_amount_sum_ratio</th>\n",
       "      <th>quarter</th>\n",
       "      <th>target</th>\n",
       "      <th>target_pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_0005b2f279</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>818</td>\n",
       "      <td>M_ID_431d369269</td>\n",
       "      <td>-11</td>\n",
       "      <td>106.0</td>\n",
       "      <td>1.485686e+09</td>\n",
       "      <td>2017</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>1.001172</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071279</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.945304</td>\n",
       "      <td>0.658630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_0005b2f279</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>278</td>\n",
       "      <td>M_ID_f064c2a380</td>\n",
       "      <td>-11</td>\n",
       "      <td>62.9</td>\n",
       "      <td>1.485695e+09</td>\n",
       "      <td>2017</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>1.001172</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071279</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.945304</td>\n",
       "      <td>1.004230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_0005b2f279</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_256a900ace</td>\n",
       "      <td>-10</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1.486204e+09</td>\n",
       "      <td>2017</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>1.001172</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071279</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.945304</td>\n",
       "      <td>0.989231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_0005b2f279</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>842</td>\n",
       "      <td>M_ID_30be6cf889</td>\n",
       "      <td>-7</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1.494040e+09</td>\n",
       "      <td>2017</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>1.001172</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071279</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.945304</td>\n",
       "      <td>0.969341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_0005b2f279</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>278</td>\n",
       "      <td>M_ID_8c0ebe57c6</td>\n",
       "      <td>-7</td>\n",
       "      <td>30.0</td>\n",
       "      <td>1.495985e+09</td>\n",
       "      <td>2017</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>1.001172</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071279</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.945304</td>\n",
       "      <td>0.706181</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 92 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  authorized_flag          card_id  installments category_3  \\\n",
       "0               1  C_ID_0005b2f279           1.0          1   \n",
       "1               1  C_ID_0005b2f279           1.0          1   \n",
       "2               1  C_ID_0005b2f279           1.0          1   \n",
       "3               1  C_ID_0005b2f279           1.0          1   \n",
       "4               1  C_ID_0005b2f279           1.0          1   \n",
       "\n",
       "  merchant_category_id      merchant_id  month_lag  purchase_amount  \\\n",
       "0                  818  M_ID_431d369269        -11            106.0   \n",
       "1                  278  M_ID_f064c2a380        -11             62.9   \n",
       "2                  307  M_ID_256a900ace        -10            100.0   \n",
       "3                  842  M_ID_30be6cf889         -7             38.0   \n",
       "4                  278  M_ID_8c0ebe57c6         -7             30.0   \n",
       "\n",
       "   purchase_date purchase_Year  ... installments_ratio month_diff_ratio  \\\n",
       "0   1.485686e+09          2017  ...           0.040816         1.001172   \n",
       "1   1.485695e+09          2017  ...           0.040816         1.001172   \n",
       "2   1.486204e+09          2017  ...           0.040816         1.001172   \n",
       "3   1.494040e+09          2017  ...           0.040816         1.001172   \n",
       "4   1.495985e+09          2017  ...           0.040816         1.001172   \n",
       "\n",
       "  new_first_buy new_last_buy observed_elapsed_time outliers  \\\n",
       "0           NaN          NaN                   NaN        0   \n",
       "1           NaN          NaN                   NaN        0   \n",
       "2           NaN          NaN                   NaN        0   \n",
       "3           NaN          NaN                   NaN        0   \n",
       "4           NaN          NaN                   NaN        0   \n",
       "\n",
       "  purchase_amount_sum_ratio quarter    target target_pred  \n",
       "0                  0.071279     2.0  2.945304    0.658630  \n",
       "1                  0.071279     2.0  2.945304    1.004230  \n",
       "2                  0.071279     2.0  2.945304    0.989231  \n",
       "3                  0.071279     2.0  2.945304    0.969341  \n",
       "4                  0.071279     2.0  2.945304    0.706181  \n",
       "\n",
       "[5 rows x 92 columns]"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_vla.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_lol = df_vla.groupby('card_id').agg({'target': ['mean'], 'target_pred': ['mean']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_lol.columns = ['_'.join(c) for c in df_lol.columns.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse(df_lol.target_mean.values, df_lol.target_pred_mean.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target_mean</th>\n",
       "      <th>target_pred_mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C_ID_fffaed41f0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_fffb67f260</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_fffd070e0e</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_fffd24dcf3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_fffd93c8bf</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 target_mean  target_pred_mean\n",
       "card_id                                       \n",
       "C_ID_fffaed41f0          NaN               NaN\n",
       "C_ID_fffb67f260          NaN               NaN\n",
       "C_ID_fffd070e0e          NaN               NaN\n",
       "C_ID_fffd24dcf3          NaN               NaN\n",
       "C_ID_fffd93c8bf          NaN               NaN"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_lol.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorized_flag</th>\n",
       "      <th>card_id</th>\n",
       "      <th>installments</th>\n",
       "      <th>category_3</th>\n",
       "      <th>merchant_category_id</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th>purchase_amount</th>\n",
       "      <th>purchase_date</th>\n",
       "      <th>purchase_Year</th>\n",
       "      <th>...</th>\n",
       "      <th>installments_ratio</th>\n",
       "      <th>month_diff_ratio</th>\n",
       "      <th>new_first_buy</th>\n",
       "      <th>new_last_buy</th>\n",
       "      <th>observed_elapsed_time</th>\n",
       "      <th>outliers</th>\n",
       "      <th>purchase_amount_sum_ratio</th>\n",
       "      <th>quarter</th>\n",
       "      <th>target</th>\n",
       "      <th>target_pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 92 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [authorized_flag, card_id, installments, category_3, merchant_category_id, merchant_id, month_lag, purchase_amount, purchase_date, purchase_Year, purchase_Month, purchase_Week, purchase_Day, purchase_Dayofweek, purchase_Dayofyear, purchase_Is_month_end, purchase_Is_month_start, purchase_Is_quarter_end, purchase_Is_quarter_start, purchase_Is_year_end, purchase_Is_year_start, purchase_Hour, purchase_Minute, purchase_Second, purchase_Elapsed, purchased_on_weekend, purchased_on_weekday, month_diff, purchase_date_successive_diff, Christmas_Day_2017, Mothers_Day_2017, fathers_day_2017, Children_day_2017, Valentine_Day_2017, Black_Friday_2017, Mothers_Day_2018, duration, amount_month_ratio, merchant_address_id, numerical_range, merchant_rating, merchant_group_id, merchant_category_id_y, subsector_id, numerical_1, numerical_2, category_1, most_recent_sales_range, most_recent_purchases_range, avg_sales_lag3, avg_purchases_lag3, active_months_lag3, avg_sales_lag6, avg_purchases_lag6, active_months_lag6, avg_sales_lag12, avg_purchases_lag12, active_months_lag12, category_4, city_id, state_id, category_2, CLV, CLV_old, CLV_ratio, card_id_count_ratio, days_feature1, days_feature1_ratio, days_feature2, days_feature2_ratio, days_feature3, days_feature3_ratio, feature_1, feature_2, feature_3, feature_max, feature_mean, feature_min, feature_sum, feature_var, hist_first_buy, hist_last_buy, installments_ratio, month_diff_ratio, new_first_buy, new_last_buy, observed_elapsed_time, outliers, purchase_amount_sum_ratio, quarter, target, target_pred]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 92 columns]"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_vla[df_vla['card_id']=='C_ID_fffd93c8bf']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2.9453 ,  0.6856 , -1.61452, ...,      nan,      nan,      nan])"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_lol.target_mean.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8951409180500303"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse(df_lol.target_mean.values, df_lol.target_pred_mean.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_lol.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_vla['target'].isnull().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorized_flag</th>\n",
       "      <th>card_id</th>\n",
       "      <th>installments</th>\n",
       "      <th>category_3</th>\n",
       "      <th>merchant_category_id</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th>purchase_amount</th>\n",
       "      <th>purchase_date</th>\n",
       "      <th>purchase_Year</th>\n",
       "      <th>...</th>\n",
       "      <th>installments_ratio</th>\n",
       "      <th>month_diff_ratio</th>\n",
       "      <th>new_first_buy</th>\n",
       "      <th>new_last_buy</th>\n",
       "      <th>observed_elapsed_time</th>\n",
       "      <th>outliers</th>\n",
       "      <th>purchase_amount_sum_ratio</th>\n",
       "      <th>quarter</th>\n",
       "      <th>target</th>\n",
       "      <th>target_pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_0005b2f279</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>818</td>\n",
       "      <td>M_ID_431d369269</td>\n",
       "      <td>-11</td>\n",
       "      <td>106.0</td>\n",
       "      <td>1.485686e+09</td>\n",
       "      <td>2017</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>1.001172</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071279</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.945304</td>\n",
       "      <td>0.658630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_0005b2f279</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>278</td>\n",
       "      <td>M_ID_f064c2a380</td>\n",
       "      <td>-11</td>\n",
       "      <td>62.9</td>\n",
       "      <td>1.485695e+09</td>\n",
       "      <td>2017</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>1.001172</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071279</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.945304</td>\n",
       "      <td>1.004230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_0005b2f279</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_256a900ace</td>\n",
       "      <td>-10</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1.486204e+09</td>\n",
       "      <td>2017</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>1.001172</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071279</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.945304</td>\n",
       "      <td>0.989231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_0005b2f279</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>842</td>\n",
       "      <td>M_ID_30be6cf889</td>\n",
       "      <td>-7</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1.494040e+09</td>\n",
       "      <td>2017</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>1.001172</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071279</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.945304</td>\n",
       "      <td>0.969341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_0005b2f279</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>278</td>\n",
       "      <td>M_ID_8c0ebe57c6</td>\n",
       "      <td>-7</td>\n",
       "      <td>30.0</td>\n",
       "      <td>1.495985e+09</td>\n",
       "      <td>2017</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>1.001172</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0.071279</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.945304</td>\n",
       "      <td>0.706181</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 92 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  authorized_flag          card_id  installments category_3  \\\n",
       "0               1  C_ID_0005b2f279           1.0          1   \n",
       "1               1  C_ID_0005b2f279           1.0          1   \n",
       "2               1  C_ID_0005b2f279           1.0          1   \n",
       "3               1  C_ID_0005b2f279           1.0          1   \n",
       "4               1  C_ID_0005b2f279           1.0          1   \n",
       "\n",
       "  merchant_category_id      merchant_id  month_lag  purchase_amount  \\\n",
       "0                  818  M_ID_431d369269        -11            106.0   \n",
       "1                  278  M_ID_f064c2a380        -11             62.9   \n",
       "2                  307  M_ID_256a900ace        -10            100.0   \n",
       "3                  842  M_ID_30be6cf889         -7             38.0   \n",
       "4                  278  M_ID_8c0ebe57c6         -7             30.0   \n",
       "\n",
       "   purchase_date purchase_Year  ... installments_ratio month_diff_ratio  \\\n",
       "0   1.485686e+09          2017  ...           0.040816         1.001172   \n",
       "1   1.485695e+09          2017  ...           0.040816         1.001172   \n",
       "2   1.486204e+09          2017  ...           0.040816         1.001172   \n",
       "3   1.494040e+09          2017  ...           0.040816         1.001172   \n",
       "4   1.495985e+09          2017  ...           0.040816         1.001172   \n",
       "\n",
       "  new_first_buy new_last_buy observed_elapsed_time outliers  \\\n",
       "0           NaN          NaN                   NaN        0   \n",
       "1           NaN          NaN                   NaN        0   \n",
       "2           NaN          NaN                   NaN        0   \n",
       "3           NaN          NaN                   NaN        0   \n",
       "4           NaN          NaN                   NaN        0   \n",
       "\n",
       "  purchase_amount_sum_ratio quarter    target target_pred  \n",
       "0                  0.071279     2.0  2.945304    0.658630  \n",
       "1                  0.071279     2.0  2.945304    1.004230  \n",
       "2                  0.071279     2.0  2.945304    0.989231  \n",
       "3                  0.071279     2.0  2.945304    0.969341  \n",
       "4                  0.071279     2.0  2.945304    0.706181  \n",
       "\n",
       "[5 rows x 92 columns]"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_vla.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}