Elo end 2 end.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Steps\" data-toc-modified-id=\"Steps-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Steps</a></span><ul class=\"toc-item\"><li><ul class=\"toc-item\"><li><span><a href=\"#Scrap-pad\" data-toc-modified-id=\"Scrap-pad-1.0.1\"><span class=\"toc-item-num\">1.0.1&nbsp;&nbsp;</span>Scrap pad</a></span></li><li><span><a href=\"#Aggregate-over-month-(not-working)\" data-toc-modified-id=\"Aggregate-over-month-(not-working)-1.0.2\"><span class=\"toc-item-num\">1.0.2&nbsp;&nbsp;</span>Aggregate over month (not working)</a></span></li><li><span><a href=\"#Load-train-&amp;-test-csvs\" data-toc-modified-id=\"Load-train-&amp;-test-csvs-1.0.3\"><span class=\"toc-item-num\">1.0.3&nbsp;&nbsp;</span>Load train &amp; test csvs</a></span></li><li><span><a href=\"#Function-to-add-extra-cols-on-train-&amp;-test-combined-agg-dfs\" data-toc-modified-id=\"Function-to-add-extra-cols-on-train-&amp;-test-combined-agg-dfs-1.0.4\"><span class=\"toc-item-num\">1.0.4&nbsp;&nbsp;</span>Function to add extra cols on train &amp; test combined agg dfs</a></span></li><li><span><a href=\"#Remove-outliers\" data-toc-modified-id=\"Remove-outliers-1.0.5\"><span class=\"toc-item-num\">1.0.5&nbsp;&nbsp;</span>Remove outliers</a></span></li><li><span><a href=\"#Target-encoding-for-outliers\" data-toc-modified-id=\"Target-encoding-for-outliers-1.0.6\"><span class=\"toc-item-num\">1.0.6&nbsp;&nbsp;</span>Target encoding for outliers</a></span></li></ul></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Steps\n",
    "\n",
    "1. Import dependencies\n",
    "2. Import processed df of new transactions\n",
    "3. Aggregate over card_id \n",
    "4. Aggregate over card_id & month_lag - for monthly aggregations of purchases, installments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(120000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 120 seconds\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 4\n",
    "%autosave 120"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.io import *\n",
    "from fastai.structured import *\n",
    "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n",
    "from pandas_summary import DataFrameSummary\n",
    "from IPython.display import display\n",
    "from sklearn import metrics\n",
    "import feather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def agg_cat(df):\n",
    "    cat_agg = {\n",
    "        'purchase_amount': ['sum', 'max', 'min', 'mean', 'std', 'var'],\n",
    "        'installments':  ['sum', 'max', 'min', 'mean', 'std', 'var']\n",
    "    }\n",
    "    intermdiate_df = (df.groupby('card_id').size().reset_index(name='transactions_count'))\n",
    "    for c in ['category_2', 'category_3']:\n",
    "        temp_df1 = df.groupby(c).agg(cat_agg)\n",
    "        temp_df1.columns = ['_'.join(i).strip() for i in temp_df1.columns.values]\n",
    "        temp_df1.reset_index(inplace=True)\n",
    "        pd.merge(intermdiate_df, temp_df1, on='card_id', how='left')\n",
    "    return intermdiate_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_extra_cols_on_agg(df):\n",
    "    df['inverse_avg_transactions_per_day'] = df['purchase_date_ptp']/df['card_id_size']\n",
    "    df['days_since_last_transaction'] = (datetime.datetime.today() - df['purchase_date_amax']).dt.days\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'new_temp' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-8e6d3d1ef3f9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcol\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'category_2'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'category_3'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mnew_temp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'_codes'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_temp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcodes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mnew_temp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'_mean'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_temp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'_codes'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'purchase_amount'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'mean'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'new_temp' is not defined"
     ]
    }
   ],
   "source": [
    "for col in ['category_2','category_3']:\n",
    "    new_temp[col + '_codes'] = new_temp[col].cat.codes\n",
    "    new_temp[col+'_mean'] = new_temp.groupby([col + '_codes'])['purchase_amount'].transform('mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def aggregate_by_card_id(df):\n",
    "    unique_cols = ['city_id', 'merchant_category_id', 'merchant_id', 'state_id', 'purchase_Month', \n",
    "                   'subsector_id']\n",
    "    aggs = {}\n",
    "    for c in unique_cols:\n",
    "        aggs[c] = ['nunique'] \n",
    "    aggs['authorized_flag'] = ['mean', 'sum']\n",
    "    aggs['purchase_amount'] = ['sum', 'max', 'min', 'mean', 'std', 'var']\n",
    "    aggs['installments'] = ['sum', 'max', 'min', 'mean', 'std', 'var']\n",
    "    aggs['month_lag'] = ['sum', 'max', 'min', 'mean', 'std', 'var']\n",
    "    aggs['purchase_date'] = [np.min, np.max , np.ptp]\n",
    "    aggs['card_id'] = ['size']\n",
    "    aggs['purchased_on_weekend'] = ['sum', 'mean']\n",
    "    aggs['month_diff'] = ['mean']\n",
    "    aggs['category_1'] = ['sum', 'mean']\n",
    "   \n",
    " #     Not working due to a bug in pandas 0.23.4\n",
    "#     cat_agg = {\n",
    "#         'purchase_amount': ['sum', 'max', 'min', 'mean', 'std', 'var'],\n",
    "#         'installments':  ['sum', 'max', 'min', 'mean', 'std', 'var']\n",
    "#     }\n",
    "#     # aggregate over cat_2, cat_3\n",
    "#     intermdiate_df = pd.DataFrame\n",
    "#     for c in ['category_2', 'category_3']:\n",
    "#         temp_df = df.groupby(c).agg(cat_agg)\n",
    "#         temp_df.columns = ['_'.join(i).strip() for i in temp_df.columns.values]\n",
    "#         temp_df.reset_index(inplace=True)\n",
    "#         pd.merge(intermdiate_df, temp_df, on='card_id', how='left')\n",
    "        \n",
    "    new_df = df.groupby(['card_id']).agg(aggs)\n",
    "    new_df.columns = ['_'.join(col).strip() for col in new_df.columns.values]\n",
    "    new_df.reset_index(inplace=True)\n",
    "    other_df = (df.groupby('card_id')\n",
    "          .size()\n",
    "          .reset_index(name='transactions_count'))\n",
    "    \n",
    "    new_df = pd.merge(other_df, new_df, on='card_id', how='left')\n",
    "    new_df[\"purchase_date_ptp\"] = new_df[\"purchase_date_ptp\"].dt.days    \n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_extra_cols(df):\n",
    "    df['purchased_on_weekend'] = (df.purchase_Dayofweek >=5).astype(int)\n",
    "    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30\n",
    "    df['month_diff'] += df['month_lag']\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mark_categorical_cols_for_transactions(df):\n",
    "    categorical_cols = ['authorized_flag', 'card_id', 'city_id', 'category_1',\n",
    "       'merchant_category_id', 'merchant_id',\n",
    "       'state_id', 'subsector_id', 'purchase_Year', 'purchase_Month']\n",
    "    for c in categorical_cols:\n",
    "        df[c] = df[c].astype('category').cat.as_ordered()\n",
    "    for col in ['authorized_flag', 'category_1']:\n",
    "        df[col] = df[col].map({'Y':1, 'N':0})\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Categorical categories must be unique",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-7205e92d6fcb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnew_merchant_transactions_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeather\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_dataframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'new_merchant_transactions_df'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pyarrow/feather.py\u001b[0m in \u001b[0;36mread_feather\u001b[0;34m(source, columns, use_threads)\u001b[0m\n\u001b[1;32m    208\u001b[0m     \"\"\"\n\u001b[1;32m    209\u001b[0m     \u001b[0mreader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFeatherReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mreader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_pandas\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_threads\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_threads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pyarrow/feather.py\u001b[0m in \u001b[0;36mread_pandas\u001b[0;34m(self, columns, use_threads)\u001b[0m\n\u001b[1;32m     70\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread_pandas\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_threads\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m         return self.read_table(columns=columns).to_pandas(\n\u001b[0;32m---> 72\u001b[0;31m             use_threads=use_threads)\n\u001b[0m\u001b[1;32m     73\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pyarrow/table.pxi\u001b[0m in \u001b[0;36mpyarrow.lib.Table.to_pandas\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pyarrow/pandas_compat.py\u001b[0m in \u001b[0;36mtable_to_blockmanager\u001b[0;34m(options, table, memory_pool, categories)\u001b[0m\n\u001b[1;32m    576\u001b[0m             )\n\u001b[1;32m    577\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 578\u001b[0;31m     \u001b[0mblocks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_table_to_blocks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock_table\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmemory_pool\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcategories\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    579\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    580\u001b[0m     \u001b[0;31m# Construct the row index\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pyarrow/pandas_compat.py\u001b[0m in \u001b[0;36m_table_to_blocks\u001b[0;34m(options, block_table, memory_pool, categories)\u001b[0m\n\u001b[1;32m    744\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    745\u001b[0m     \u001b[0;31m# Defined above\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 746\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_reconstruct_block\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mitem\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    747\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    748\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pyarrow/pandas_compat.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    744\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    745\u001b[0m     \u001b[0;31m# Defined above\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 746\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_reconstruct_block\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mitem\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    747\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    748\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pyarrow/pandas_compat.py\u001b[0m in \u001b[0;36m_reconstruct_block\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m    475\u001b[0m         cat = pd.Categorical.from_codes(block_arr,\n\u001b[1;32m    476\u001b[0m                                         \u001b[0mcategories\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'dictionary'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m                                         ordered=item['ordered'])\n\u001b[0m\u001b[1;32m    478\u001b[0m         block = _int.make_block(cat, placement=placement,\n\u001b[1;32m    479\u001b[0m                                 klass=_int.CategoricalBlock)\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/categorical.py\u001b[0m in \u001b[0;36mfrom_codes\u001b[0;34m(cls, codes, categories, ordered)\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/dtypes/dtypes.py\u001b[0m in \u001b[0;36m_validate_categories\u001b[0;34m(categories, fastpath)\u001b[0m\n\u001b[1;32m    325\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcategories\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mABCCategoricalIndex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 327\u001b[0;31m             \u001b[0mcategories\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcategories\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcategories\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    328\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    329\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcategories\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Categorical categories must be unique"
     ]
    }
   ],
   "source": [
    "new_merchant_transactions_df = feather.read_dataframe('new_merchant_transactions_df')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_merchant_transactions_df = add_extra_cols(new_merchant_transactions_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_merchant_transactions_df = mark_categorical_cols_for_transactions(new_merchant_transactions_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 50.2 s, sys: 978 ms, total: 51.2 s\n",
      "Wall time: 36.6 s\n"
     ]
    }
   ],
   "source": [
    "%time new_hist_agg = aggregate_by_card_id(new_merchant_transactions_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_merchant_transactions_df.to_feather('new_merchant_transactions_df')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_hist_agg[\"purchase_date_ptp\"] = new_hist_agg[\"purchase_date_ptp\"].dt.days    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_hist_agg.to_feather('new_hist_agg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_hist_agg = feather.read_dataframe('new_hist_agg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_hist_agg = add_extra_cols_on_agg(new_hist_agg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Scrap pad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "temp_df = new_merchant_transactions_df[20:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "temp_df_agg = temp_df.groupby(['card_id','month_lag']).agg({\"purchase_amount\": ['min', 'max', 'mean', 'std']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([('purchase_amount', 'min'), ('purchase_amount', 'max'), ('purchase_amount', 'mean'),\n",
       "       ('purchase_amount', 'std')], dtype=object)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df_agg.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['purchase_amount_min',\n",
       " 'purchase_amount_max',\n",
       " 'purchase_amount_mean',\n",
       " 'purchase_amount_std']"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "['_'.join(c).strip() for c in temp_df_agg.columns.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\">purchase_amount</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C_ID_241a01e9d9</th>\n",
       "      <th>2</th>\n",
       "      <td>-0.461705</td>\n",
       "      <td>-0.294760</td>\n",
       "      <td>-0.378232</td>\n",
       "      <td>0.118048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_a97720321f</th>\n",
       "      <th>1</th>\n",
       "      <td>-0.202377</td>\n",
       "      <td>0.755743</td>\n",
       "      <td>0.276683</td>\n",
       "      <td>0.677493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">C_ID_ef55cf8d4b</th>\n",
       "      <th>1</th>\n",
       "      <td>-0.704834</td>\n",
       "      <td>3.009720</td>\n",
       "      <td>0.683685</td>\n",
       "      <td>1.753241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.432253</td>\n",
       "      <td>-0.432253</td>\n",
       "      <td>-0.432253</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          purchase_amount                              \n",
       "                                      min       max      mean       std\n",
       "card_id         month_lag                                              \n",
       "C_ID_241a01e9d9 2               -0.461705 -0.294760 -0.378232  0.118048\n",
       "C_ID_a97720321f 1               -0.202377  0.755743  0.276683  0.677493\n",
       "C_ID_ef55cf8d4b 1               -0.704834  3.009720  0.683685  1.753241\n",
       "                2               -0.432253 -0.432253 -0.432253       NaN"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df_agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "temp_df_agg.columns = ['_'.join(c).strip() for c in temp_df_agg.columns.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>purchase_amount_min</th>\n",
       "      <th>purchase_amount_max</th>\n",
       "      <th>purchase_amount_mean</th>\n",
       "      <th>purchase_amount_std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C_ID_241a01e9d9</th>\n",
       "      <th>2</th>\n",
       "      <td>-0.461705</td>\n",
       "      <td>-0.294760</td>\n",
       "      <td>-0.378232</td>\n",
       "      <td>0.118048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_a97720321f</th>\n",
       "      <th>1</th>\n",
       "      <td>-0.202377</td>\n",
       "      <td>0.755743</td>\n",
       "      <td>0.276683</td>\n",
       "      <td>0.677493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">C_ID_ef55cf8d4b</th>\n",
       "      <th>1</th>\n",
       "      <td>-0.704834</td>\n",
       "      <td>3.009720</td>\n",
       "      <td>0.683685</td>\n",
       "      <td>1.753241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.432253</td>\n",
       "      <td>-0.432253</td>\n",
       "      <td>-0.432253</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           purchase_amount_min  purchase_amount_max  \\\n",
       "card_id         month_lag                                             \n",
       "C_ID_241a01e9d9 2                    -0.461705            -0.294760   \n",
       "C_ID_a97720321f 1                    -0.202377             0.755743   \n",
       "C_ID_ef55cf8d4b 1                    -0.704834             3.009720   \n",
       "                2                    -0.432253            -0.432253   \n",
       "\n",
       "                           purchase_amount_mean  purchase_amount_std  \n",
       "card_id         month_lag                                             \n",
       "C_ID_241a01e9d9 2                     -0.378232             0.118048  \n",
       "C_ID_a97720321f 1                      0.276683             0.677493  \n",
       "C_ID_ef55cf8d4b 1                      0.683685             1.753241  \n",
       "                2                     -0.432253                  NaN  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df_agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">purchase_amount_min</th>\n",
       "      <th colspan=\"2\" halign=\"left\">purchase_amount_max</th>\n",
       "      <th colspan=\"2\" halign=\"left\">purchase_amount_mean</th>\n",
       "      <th colspan=\"2\" halign=\"left\">purchase_amount_std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C_ID_241a01e9d9</th>\n",
       "      <td>-0.461705</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.294760</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.378232</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.118048</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_a97720321f</th>\n",
       "      <td>-0.202377</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.755743</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.276683</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.677493</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID_ef55cf8d4b</th>\n",
       "      <td>-0.568543</td>\n",
       "      <td>0.192744</td>\n",
       "      <td>1.288734</td>\n",
       "      <td>2.433842</td>\n",
       "      <td>0.125716</td>\n",
       "      <td>0.789087</td>\n",
       "      <td>1.753241</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                purchase_amount_min           purchase_amount_max            \\\n",
       "                               mean       std                mean       std   \n",
       "card_id                                                                       \n",
       "C_ID_241a01e9d9           -0.461705       NaN           -0.294760       NaN   \n",
       "C_ID_a97720321f           -0.202377       NaN            0.755743       NaN   \n",
       "C_ID_ef55cf8d4b           -0.568543  0.192744            1.288734  2.433842   \n",
       "\n",
       "                purchase_amount_mean           purchase_amount_std      \n",
       "                                mean       std                mean std  \n",
       "card_id                                                                 \n",
       "C_ID_241a01e9d9            -0.378232       NaN            0.118048 NaN  \n",
       "C_ID_a97720321f             0.276683       NaN            0.677493 NaN  \n",
       "C_ID_ef55cf8d4b             0.125716  0.789087            1.753241 NaN  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df_agg.groupby('card_id').agg(['mean', 'std'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Aggregate over month (not working)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def aggregate_over_month(df):\n",
    "    grouped = df.groupby(['card_id','month_lag'])\n",
    "    agg_func = {\n",
    "        'purchase_amount': ['count', 'sum', 'mean', 'min', 'max', 'std'],\n",
    "        'installments': ['count', 'sum', 'mean', 'min', 'max', 'std'],\n",
    "        }\n",
    "    temp_group = grouped.agg(agg_func)\n",
    "    temp_group.columns = ['_'.join(c).strip() for c in temp_group.columns.values]\n",
    "    temp_group.reset_index(inplace=True)\n",
    "    final_group = temp_group.groupby('card_id').agg(['mean', 'std'])\n",
    "    final_group.columns = [\"_\".join(c).strip() for c in final_group.columns.values]\n",
    "    final_group.reset_index(inplace=True)\n",
    "    return final_group\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x = aggregate_over_month(temp_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def aggregate_per_month(history):\n",
    "    grouped = history.groupby(['card_id', 'month_lag'])\n",
    "\n",
    "    agg_func = {\n",
    "            'purchase_amount': ['count', 'sum', 'mean', 'min', 'max', 'std'],\n",
    "            'installments': ['count', 'sum', 'mean', 'min', 'max', 'std'],\n",
    "            }\n",
    "\n",
    "    intermediate_group = grouped.agg(agg_func)\n",
    "    intermediate_group.columns = ['_'.join(col).strip() for col in intermediate_group.columns.values]\n",
    "    intermediate_group.reset_index(inplace=True)\n",
    "\n",
    "    final_group = intermediate_group.groupby('card_id').agg(['mean', 'std'])\n",
    "    final_group.columns = ['_'.join(col).strip() for col in final_group.columns.values]\n",
    "    final_group.reset_index(inplace=True)\n",
    "    \n",
    "    return final_group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x = aggregate_per_month(temp_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>card_id</th>\n",
       "      <th>month_lag_mean</th>\n",
       "      <th>month_lag_std</th>\n",
       "      <th>purchase_amount_count_mean</th>\n",
       "      <th>purchase_amount_count_std</th>\n",
       "      <th>purchase_amount_sum_mean</th>\n",
       "      <th>purchase_amount_sum_std</th>\n",
       "      <th>purchase_amount_mean_mean</th>\n",
       "      <th>purchase_amount_mean_std</th>\n",
       "      <th>purchase_amount_min_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>installments_sum_mean</th>\n",
       "      <th>installments_sum_std</th>\n",
       "      <th>installments_mean_mean</th>\n",
       "      <th>installments_mean_std</th>\n",
       "      <th>installments_min_mean</th>\n",
       "      <th>installments_min_std</th>\n",
       "      <th>installments_max_mean</th>\n",
       "      <th>installments_max_std</th>\n",
       "      <th>installments_std_mean</th>\n",
       "      <th>installments_std_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C_ID_0001238066</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C_ID_0001506ef0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C_ID_0001793786</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C_ID_000183fdda</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C_ID_0002709b5a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>C_ID_000298032a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>C_ID_0002ba3c2e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>C_ID_0002c7c2c1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>C_ID_00032df08f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>C_ID_0003754056</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>C_ID_000377f6a0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>C_ID_0003be3c83</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>C_ID_0003f41435</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>C_ID_00042d509c</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>C_ID_0004587331</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>C_ID_0004725b87</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>C_ID_0004888ddd</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>C_ID_0004b68c49</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>C_ID_0004c2a5ab</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>C_ID_00057b99fe</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>C_ID_000599daf9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>C_ID_0005f16cc8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>C_ID_0006152db8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>C_ID_000616f4a8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>C_ID_000664aa02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>C_ID_00079fec55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>C_ID_0007a60a33</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>C_ID_0007fe3157</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>C_ID_000844d8f1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289971</th>\n",
       "      <td>C_ID_fffaed41f0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289972</th>\n",
       "      <td>C_ID_fffb79fb56</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289973</th>\n",
       "      <td>C_ID_fffba72dc5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289974</th>\n",
       "      <td>C_ID_fffbdf036b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289975</th>\n",
       "      <td>C_ID_fffbee5c24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289976</th>\n",
       "      <td>C_ID_fffc96bf24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289977</th>\n",
       "      <td>C_ID_fffcb74f49</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289978</th>\n",
       "      <td>C_ID_fffcf66e12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289979</th>\n",
       "      <td>C_ID_fffd070e0e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289980</th>\n",
       "      <td>C_ID_fffd1207a8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289981</th>\n",
       "      <td>C_ID_fffd24dcf3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289982</th>\n",
       "      <td>C_ID_fffd337484</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289983</th>\n",
       "      <td>C_ID_fffd345048</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289984</th>\n",
       "      <td>C_ID_fffd351e7f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289985</th>\n",
       "      <td>C_ID_fffd93c8bf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289986</th>\n",
       "      <td>C_ID_fffd943b91</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289987</th>\n",
       "      <td>C_ID_fffdad8aa0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289988</th>\n",
       "      <td>C_ID_fffde15ab6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289989</th>\n",
       "      <td>C_ID_fffdf34798</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289990</th>\n",
       "      <td>C_ID_fffe29a04a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289991</th>\n",
       "      <td>C_ID_fffe78b232</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289992</th>\n",
       "      <td>C_ID_fffe866457</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289993</th>\n",
       "      <td>C_ID_fffea6de74</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289994</th>\n",
       "      <td>C_ID_fffeced303</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289995</th>\n",
       "      <td>C_ID_fffeed3a89</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289996</th>\n",
       "      <td>C_ID_ffff1d9928</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289997</th>\n",
       "      <td>C_ID_ffff579d3a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289998</th>\n",
       "      <td>C_ID_ffff756266</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289999</th>\n",
       "      <td>C_ID_ffff828181</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290000</th>\n",
       "      <td>C_ID_fffffd5772</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>290001 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                card_id  month_lag_mean  month_lag_std  \\\n",
       "0       C_ID_00007093c1             NaN            NaN   \n",
       "1       C_ID_0001238066             NaN            NaN   \n",
       "2       C_ID_0001506ef0             NaN            NaN   \n",
       "3       C_ID_0001793786             NaN            NaN   \n",
       "4       C_ID_000183fdda             NaN            NaN   \n",
       "5       C_ID_0002709b5a             NaN            NaN   \n",
       "6       C_ID_000298032a             NaN            NaN   \n",
       "7       C_ID_0002ba3c2e             NaN            NaN   \n",
       "8       C_ID_0002c7c2c1             NaN            NaN   \n",
       "9       C_ID_00032df08f             NaN            NaN   \n",
       "10      C_ID_0003754056             NaN            NaN   \n",
       "11      C_ID_000377f6a0             NaN            NaN   \n",
       "12      C_ID_0003be3c83             NaN            NaN   \n",
       "13      C_ID_0003f41435             NaN            NaN   \n",
       "14      C_ID_00042d509c             NaN            NaN   \n",
       "15      C_ID_0004587331             NaN            NaN   \n",
       "16      C_ID_0004725b87             NaN            NaN   \n",
       "17      C_ID_0004888ddd             NaN            NaN   \n",
       "18      C_ID_0004b68c49             NaN            NaN   \n",
       "19      C_ID_0004c2a5ab             NaN            NaN   \n",
       "20      C_ID_00057b99fe             NaN            NaN   \n",
       "21      C_ID_000599daf9             NaN            NaN   \n",
       "22      C_ID_0005f16cc8             NaN            NaN   \n",
       "23      C_ID_0006152db8             NaN            NaN   \n",
       "24      C_ID_000616f4a8             NaN            NaN   \n",
       "25      C_ID_000664aa02             NaN            NaN   \n",
       "26      C_ID_00079fec55             NaN            NaN   \n",
       "27      C_ID_0007a60a33             NaN            NaN   \n",
       "28      C_ID_0007fe3157             NaN            NaN   \n",
       "29      C_ID_000844d8f1             NaN            NaN   \n",
       "...                 ...             ...            ...   \n",
       "289971  C_ID_fffaed41f0             NaN            NaN   \n",
       "289972  C_ID_fffb79fb56             NaN            NaN   \n",
       "289973  C_ID_fffba72dc5             NaN            NaN   \n",
       "289974  C_ID_fffbdf036b             NaN            NaN   \n",
       "289975  C_ID_fffbee5c24             NaN            NaN   \n",
       "289976  C_ID_fffc96bf24             NaN            NaN   \n",
       "289977  C_ID_fffcb74f49             NaN            NaN   \n",
       "289978  C_ID_fffcf66e12             NaN            NaN   \n",
       "289979  C_ID_fffd070e0e             NaN            NaN   \n",
       "289980  C_ID_fffd1207a8             NaN            NaN   \n",
       "289981  C_ID_fffd24dcf3             NaN            NaN   \n",
       "289982  C_ID_fffd337484             NaN            NaN   \n",
       "289983  C_ID_fffd345048             NaN            NaN   \n",
       "289984  C_ID_fffd351e7f             NaN            NaN   \n",
       "289985  C_ID_fffd93c8bf             NaN            NaN   \n",
       "289986  C_ID_fffd943b91             NaN            NaN   \n",
       "289987  C_ID_fffdad8aa0             NaN            NaN   \n",
       "289988  C_ID_fffde15ab6             NaN            NaN   \n",
       "289989  C_ID_fffdf34798             NaN            NaN   \n",
       "289990  C_ID_fffe29a04a             NaN            NaN   \n",
       "289991  C_ID_fffe78b232             NaN            NaN   \n",
       "289992  C_ID_fffe866457             NaN            NaN   \n",
       "289993  C_ID_fffea6de74             NaN            NaN   \n",
       "289994  C_ID_fffeced303             NaN            NaN   \n",
       "289995  C_ID_fffeed3a89             NaN            NaN   \n",
       "289996  C_ID_ffff1d9928             NaN            NaN   \n",
       "289997  C_ID_ffff579d3a             NaN            NaN   \n",
       "289998  C_ID_ffff756266             NaN            NaN   \n",
       "289999  C_ID_ffff828181             NaN            NaN   \n",
       "290000  C_ID_fffffd5772             NaN            NaN   \n",
       "\n",
       "        purchase_amount_count_mean  purchase_amount_count_std  \\\n",
       "0                              NaN                        NaN   \n",
       "1                              NaN                        NaN   \n",
       "2                              NaN                        NaN   \n",
       "3                              NaN                        NaN   \n",
       "4                              NaN                        NaN   \n",
       "5                              NaN                        NaN   \n",
       "6                              NaN                        NaN   \n",
       "7                              NaN                        NaN   \n",
       "8                              NaN                        NaN   \n",
       "9                              NaN                        NaN   \n",
       "10                             NaN                        NaN   \n",
       "11                             NaN                        NaN   \n",
       "12                             NaN                        NaN   \n",
       "13                             NaN                        NaN   \n",
       "14                             NaN                        NaN   \n",
       "15                             NaN                        NaN   \n",
       "16                             NaN                        NaN   \n",
       "17                             NaN                        NaN   \n",
       "18                             NaN                        NaN   \n",
       "19                             NaN                        NaN   \n",
       "20                             NaN                        NaN   \n",
       "21                             NaN                        NaN   \n",
       "22                             NaN                        NaN   \n",
       "23                             NaN                        NaN   \n",
       "24                             NaN                        NaN   \n",
       "25                             NaN                        NaN   \n",
       "26                             NaN                        NaN   \n",
       "27                             NaN                        NaN   \n",
       "28                             NaN                        NaN   \n",
       "29                             NaN                        NaN   \n",
       "...                            ...                        ...   \n",
       "289971                         NaN                        NaN   \n",
       "289972                         NaN                        NaN   \n",
       "289973                         NaN                        NaN   \n",
       "289974                         NaN                        NaN   \n",
       "289975                         NaN                        NaN   \n",
       "289976                         NaN                        NaN   \n",
       "289977                         NaN                        NaN   \n",
       "289978                         NaN                        NaN   \n",
       "289979                         NaN                        NaN   \n",
       "289980                         NaN                        NaN   \n",
       "289981                         NaN                        NaN   \n",
       "289982                         NaN                        NaN   \n",
       "289983                         NaN                        NaN   \n",
       "289984                         NaN                        NaN   \n",
       "289985                         NaN                        NaN   \n",
       "289986                         NaN                        NaN   \n",
       "289987                         NaN                        NaN   \n",
       "289988                         NaN                        NaN   \n",
       "289989                         NaN                        NaN   \n",
       "289990                         NaN                        NaN   \n",
       "289991                         NaN                        NaN   \n",
       "289992                         NaN                        NaN   \n",
       "289993                         NaN                        NaN   \n",
       "289994                         NaN                        NaN   \n",
       "289995                         NaN                        NaN   \n",
       "289996                         NaN                        NaN   \n",
       "289997                         NaN                        NaN   \n",
       "289998                         NaN                        NaN   \n",
       "289999                         NaN                        NaN   \n",
       "290000                         NaN                        NaN   \n",
       "\n",
       "        purchase_amount_sum_mean  purchase_amount_sum_std  \\\n",
       "0                            NaN                      NaN   \n",
       "1                            NaN                      NaN   \n",
       "2                            NaN                      NaN   \n",
       "3                            NaN                      NaN   \n",
       "4                            NaN                      NaN   \n",
       "5                            NaN                      NaN   \n",
       "6                            NaN                      NaN   \n",
       "7                            NaN                      NaN   \n",
       "8                            NaN                      NaN   \n",
       "9                            NaN                      NaN   \n",
       "10                           NaN                      NaN   \n",
       "11                           NaN                      NaN   \n",
       "12                           NaN                      NaN   \n",
       "13                           NaN                      NaN   \n",
       "14                           NaN                      NaN   \n",
       "15                           NaN                      NaN   \n",
       "16                           NaN                      NaN   \n",
       "17                           NaN                      NaN   \n",
       "18                           NaN                      NaN   \n",
       "19                           NaN                      NaN   \n",
       "20                           NaN                      NaN   \n",
       "21                           NaN                      NaN   \n",
       "22                           NaN                      NaN   \n",
       "23                           NaN                      NaN   \n",
       "24                           NaN                      NaN   \n",
       "25                           NaN                      NaN   \n",
       "26                           NaN                      NaN   \n",
       "27                           NaN                      NaN   \n",
       "28                           NaN                      NaN   \n",
       "29                           NaN                      NaN   \n",
       "...                          ...                      ...   \n",
       "289971                       NaN                      NaN   \n",
       "289972                       NaN                      NaN   \n",
       "289973                       NaN                      NaN   \n",
       "289974                       NaN                      NaN   \n",
       "289975                       NaN                      NaN   \n",
       "289976                       NaN                      NaN   \n",
       "289977                       NaN                      NaN   \n",
       "289978                       NaN                      NaN   \n",
       "289979                       NaN                      NaN   \n",
       "289980                       NaN                      NaN   \n",
       "289981                       NaN                      NaN   \n",
       "289982                       NaN                      NaN   \n",
       "289983                       NaN                      NaN   \n",
       "289984                       NaN                      NaN   \n",
       "289985                       NaN                      NaN   \n",
       "289986                       NaN                      NaN   \n",
       "289987                       NaN                      NaN   \n",
       "289988                       NaN                      NaN   \n",
       "289989                       NaN                      NaN   \n",
       "289990                       NaN                      NaN   \n",
       "289991                       NaN                      NaN   \n",
       "289992                       NaN                      NaN   \n",
       "289993                       NaN                      NaN   \n",
       "289994                       NaN                      NaN   \n",
       "289995                       NaN                      NaN   \n",
       "289996                       NaN                      NaN   \n",
       "289997                       NaN                      NaN   \n",
       "289998                       NaN                      NaN   \n",
       "289999                       NaN                      NaN   \n",
       "290000                       NaN                      NaN   \n",
       "\n",
       "        purchase_amount_mean_mean  purchase_amount_mean_std  \\\n",
       "0                             NaN                       NaN   \n",
       "1                             NaN                       NaN   \n",
       "2                             NaN                       NaN   \n",
       "3                             NaN                       NaN   \n",
       "4                             NaN                       NaN   \n",
       "5                             NaN                       NaN   \n",
       "6                             NaN                       NaN   \n",
       "7                             NaN                       NaN   \n",
       "8                             NaN                       NaN   \n",
       "9                             NaN                       NaN   \n",
       "10                            NaN                       NaN   \n",
       "11                            NaN                       NaN   \n",
       "12                            NaN                       NaN   \n",
       "13                            NaN                       NaN   \n",
       "14                            NaN                       NaN   \n",
       "15                            NaN                       NaN   \n",
       "16                            NaN                       NaN   \n",
       "17                            NaN                       NaN   \n",
       "18                            NaN                       NaN   \n",
       "19                            NaN                       NaN   \n",
       "20                            NaN                       NaN   \n",
       "21                            NaN                       NaN   \n",
       "22                            NaN                       NaN   \n",
       "23                            NaN                       NaN   \n",
       "24                            NaN                       NaN   \n",
       "25                            NaN                       NaN   \n",
       "26                            NaN                       NaN   \n",
       "27                            NaN                       NaN   \n",
       "28                            NaN                       NaN   \n",
       "29                            NaN                       NaN   \n",
       "...                           ...                       ...   \n",
       "289971                        NaN                       NaN   \n",
       "289972                        NaN                       NaN   \n",
       "289973                        NaN                       NaN   \n",
       "289974                        NaN                       NaN   \n",
       "289975                        NaN                       NaN   \n",
       "289976                        NaN                       NaN   \n",
       "289977                        NaN                       NaN   \n",
       "289978                        NaN                       NaN   \n",
       "289979                        NaN                       NaN   \n",
       "289980                        NaN                       NaN   \n",
       "289981                        NaN                       NaN   \n",
       "289982                        NaN                       NaN   \n",
       "289983                        NaN                       NaN   \n",
       "289984                        NaN                       NaN   \n",
       "289985                        NaN                       NaN   \n",
       "289986                        NaN                       NaN   \n",
       "289987                        NaN                       NaN   \n",
       "289988                        NaN                       NaN   \n",
       "289989                        NaN                       NaN   \n",
       "289990                        NaN                       NaN   \n",
       "289991                        NaN                       NaN   \n",
       "289992                        NaN                       NaN   \n",
       "289993                        NaN                       NaN   \n",
       "289994                        NaN                       NaN   \n",
       "289995                        NaN                       NaN   \n",
       "289996                        NaN                       NaN   \n",
       "289997                        NaN                       NaN   \n",
       "289998                        NaN                       NaN   \n",
       "289999                        NaN                       NaN   \n",
       "290000                        NaN                       NaN   \n",
       "\n",
       "        purchase_amount_min_mean          ...           installments_sum_mean  \\\n",
       "0                            NaN          ...                             NaN   \n",
       "1                            NaN          ...                             NaN   \n",
       "2                            NaN          ...                             NaN   \n",
       "3                            NaN          ...                             NaN   \n",
       "4                            NaN          ...                             NaN   \n",
       "5                            NaN          ...                             NaN   \n",
       "6                            NaN          ...                             NaN   \n",
       "7                            NaN          ...                             NaN   \n",
       "8                            NaN          ...                             NaN   \n",
       "9                            NaN          ...                             NaN   \n",
       "10                           NaN          ...                             NaN   \n",
       "11                           NaN          ...                             NaN   \n",
       "12                           NaN          ...                             NaN   \n",
       "13                           NaN          ...                             NaN   \n",
       "14                           NaN          ...                             NaN   \n",
       "15                           NaN          ...                             NaN   \n",
       "16                           NaN          ...                             NaN   \n",
       "17                           NaN          ...                             NaN   \n",
       "18                           NaN          ...                             NaN   \n",
       "19                           NaN          ...                             NaN   \n",
       "20                           NaN          ...                             NaN   \n",
       "21                           NaN          ...                             NaN   \n",
       "22                           NaN          ...                             NaN   \n",
       "23                           NaN          ...                             NaN   \n",
       "24                           NaN          ...                             NaN   \n",
       "25                           NaN          ...                             NaN   \n",
       "26                           NaN          ...                             NaN   \n",
       "27                           NaN          ...                             NaN   \n",
       "28                           NaN          ...                             NaN   \n",
       "29                           NaN          ...                             NaN   \n",
       "...                          ...          ...                             ...   \n",
       "289971                       NaN          ...                             NaN   \n",
       "289972                       NaN          ...                             NaN   \n",
       "289973                       NaN          ...                             NaN   \n",
       "289974                       NaN          ...                             NaN   \n",
       "289975                       NaN          ...                             NaN   \n",
       "289976                       NaN          ...                             NaN   \n",
       "289977                       NaN          ...                             NaN   \n",
       "289978                       NaN          ...                             NaN   \n",
       "289979                       NaN          ...                             NaN   \n",
       "289980                       NaN          ...                             NaN   \n",
       "289981                       NaN          ...                             NaN   \n",
       "289982                       NaN          ...                             NaN   \n",
       "289983                       NaN          ...                             NaN   \n",
       "289984                       NaN          ...                             NaN   \n",
       "289985                       NaN          ...                             NaN   \n",
       "289986                       NaN          ...                             NaN   \n",
       "289987                       NaN          ...                             NaN   \n",
       "289988                       NaN          ...                             NaN   \n",
       "289989                       NaN          ...                             NaN   \n",
       "289990                       NaN          ...                             NaN   \n",
       "289991                       NaN          ...                             NaN   \n",
       "289992                       NaN          ...                             NaN   \n",
       "289993                       NaN          ...                             NaN   \n",
       "289994                       NaN          ...                             NaN   \n",
       "289995                       NaN          ...                             NaN   \n",
       "289996                       NaN          ...                             NaN   \n",
       "289997                       NaN          ...                             NaN   \n",
       "289998                       NaN          ...                             NaN   \n",
       "289999                       NaN          ...                             NaN   \n",
       "290000                       NaN          ...                             NaN   \n",
       "\n",
       "        installments_sum_std  installments_mean_mean  installments_mean_std  \\\n",
       "0                        NaN                     NaN                    NaN   \n",
       "1                        NaN                     NaN                    NaN   \n",
       "2                        NaN                     NaN                    NaN   \n",
       "3                        NaN                     NaN                    NaN   \n",
       "4                        NaN                     NaN                    NaN   \n",
       "5                        NaN                     NaN                    NaN   \n",
       "6                        NaN                     NaN                    NaN   \n",
       "7                        NaN                     NaN                    NaN   \n",
       "8                        NaN                     NaN                    NaN   \n",
       "9                        NaN                     NaN                    NaN   \n",
       "10                       NaN                     NaN                    NaN   \n",
       "11                       NaN                     NaN                    NaN   \n",
       "12                       NaN                     NaN                    NaN   \n",
       "13                       NaN                     NaN                    NaN   \n",
       "14                       NaN                     NaN                    NaN   \n",
       "15                       NaN                     NaN                    NaN   \n",
       "16                       NaN                     NaN                    NaN   \n",
       "17                       NaN                     NaN                    NaN   \n",
       "18                       NaN                     NaN                    NaN   \n",
       "19                       NaN                     NaN                    NaN   \n",
       "20                       NaN                     NaN                    NaN   \n",
       "21                       NaN                     NaN                    NaN   \n",
       "22                       NaN                     NaN                    NaN   \n",
       "23                       NaN                     NaN                    NaN   \n",
       "24                       NaN                     NaN                    NaN   \n",
       "25                       NaN                     NaN                    NaN   \n",
       "26                       NaN                     NaN                    NaN   \n",
       "27                       NaN                     NaN                    NaN   \n",
       "28                       NaN                     NaN                    NaN   \n",
       "29                       NaN                     NaN                    NaN   \n",
       "...                      ...                     ...                    ...   \n",
       "289971                   NaN                     NaN                    NaN   \n",
       "289972                   NaN                     NaN                    NaN   \n",
       "289973                   NaN                     NaN                    NaN   \n",
       "289974                   NaN                     NaN                    NaN   \n",
       "289975                   NaN                     NaN                    NaN   \n",
       "289976                   NaN                     NaN                    NaN   \n",
       "289977                   NaN                     NaN                    NaN   \n",
       "289978                   NaN                     NaN                    NaN   \n",
       "289979                   NaN                     NaN                    NaN   \n",
       "289980                   NaN                     NaN                    NaN   \n",
       "289981                   NaN                     NaN                    NaN   \n",
       "289982                   NaN                     NaN                    NaN   \n",
       "289983                   NaN                     NaN                    NaN   \n",
       "289984                   NaN                     NaN                    NaN   \n",
       "289985                   NaN                     NaN                    NaN   \n",
       "289986                   NaN                     NaN                    NaN   \n",
       "289987                   NaN                     NaN                    NaN   \n",
       "289988                   NaN                     NaN                    NaN   \n",
       "289989                   NaN                     NaN                    NaN   \n",
       "289990                   NaN                     NaN                    NaN   \n",
       "289991                   NaN                     NaN                    NaN   \n",
       "289992                   NaN                     NaN                    NaN   \n",
       "289993                   NaN                     NaN                    NaN   \n",
       "289994                   NaN                     NaN                    NaN   \n",
       "289995                   NaN                     NaN                    NaN   \n",
       "289996                   NaN                     NaN                    NaN   \n",
       "289997                   NaN                     NaN                    NaN   \n",
       "289998                   NaN                     NaN                    NaN   \n",
       "289999                   NaN                     NaN                    NaN   \n",
       "290000                   NaN                     NaN                    NaN   \n",
       "\n",
       "        installments_min_mean  installments_min_std  installments_max_mean  \\\n",
       "0                         NaN                   NaN                    NaN   \n",
       "1                         NaN                   NaN                    NaN   \n",
       "2                         NaN                   NaN                    NaN   \n",
       "3                         NaN                   NaN                    NaN   \n",
       "4                         NaN                   NaN                    NaN   \n",
       "5                         NaN                   NaN                    NaN   \n",
       "6                         NaN                   NaN                    NaN   \n",
       "7                         NaN                   NaN                    NaN   \n",
       "8                         NaN                   NaN                    NaN   \n",
       "9                         NaN                   NaN                    NaN   \n",
       "10                        NaN                   NaN                    NaN   \n",
       "11                        NaN                   NaN                    NaN   \n",
       "12                        NaN                   NaN                    NaN   \n",
       "13                        NaN                   NaN                    NaN   \n",
       "14                        NaN                   NaN                    NaN   \n",
       "15                        NaN                   NaN                    NaN   \n",
       "16                        NaN                   NaN                    NaN   \n",
       "17                        NaN                   NaN                    NaN   \n",
       "18                        NaN                   NaN                    NaN   \n",
       "19                        NaN                   NaN                    NaN   \n",
       "20                        NaN                   NaN                    NaN   \n",
       "21                        NaN                   NaN                    NaN   \n",
       "22                        NaN                   NaN                    NaN   \n",
       "23                        NaN                   NaN                    NaN   \n",
       "24                        NaN                   NaN                    NaN   \n",
       "25                        NaN                   NaN                    NaN   \n",
       "26                        NaN                   NaN                    NaN   \n",
       "27                        NaN                   NaN                    NaN   \n",
       "28                        NaN                   NaN                    NaN   \n",
       "29                        NaN                   NaN                    NaN   \n",
       "...                       ...                   ...                    ...   \n",
       "289971                    NaN                   NaN                    NaN   \n",
       "289972                    NaN                   NaN                    NaN   \n",
       "289973                    NaN                   NaN                    NaN   \n",
       "289974                    NaN                   NaN                    NaN   \n",
       "289975                    NaN                   NaN                    NaN   \n",
       "289976                    NaN                   NaN                    NaN   \n",
       "289977                    NaN                   NaN                    NaN   \n",
       "289978                    NaN                   NaN                    NaN   \n",
       "289979                    NaN                   NaN                    NaN   \n",
       "289980                    NaN                   NaN                    NaN   \n",
       "289981                    NaN                   NaN                    NaN   \n",
       "289982                    NaN                   NaN                    NaN   \n",
       "289983                    NaN                   NaN                    NaN   \n",
       "289984                    NaN                   NaN                    NaN   \n",
       "289985                    NaN                   NaN                    NaN   \n",
       "289986                    NaN                   NaN                    NaN   \n",
       "289987                    NaN                   NaN                    NaN   \n",
       "289988                    NaN                   NaN                    NaN   \n",
       "289989                    NaN                   NaN                    NaN   \n",
       "289990                    NaN                   NaN                    NaN   \n",
       "289991                    NaN                   NaN                    NaN   \n",
       "289992                    NaN                   NaN                    NaN   \n",
       "289993                    NaN                   NaN                    NaN   \n",
       "289994                    NaN                   NaN                    NaN   \n",
       "289995                    NaN                   NaN                    NaN   \n",
       "289996                    NaN                   NaN                    NaN   \n",
       "289997                    NaN                   NaN                    NaN   \n",
       "289998                    NaN                   NaN                    NaN   \n",
       "289999                    NaN                   NaN                    NaN   \n",
       "290000                    NaN                   NaN                    NaN   \n",
       "\n",
       "        installments_max_std  installments_std_mean  installments_std_std  \n",
       "0                        NaN                    NaN                   NaN  \n",
       "1                        NaN                    NaN                   NaN  \n",
       "2                        NaN                    NaN                   NaN  \n",
       "3                        NaN                    NaN                   NaN  \n",
       "4                        NaN                    NaN                   NaN  \n",
       "5                        NaN                    NaN                   NaN  \n",
       "6                        NaN                    NaN                   NaN  \n",
       "7                        NaN                    NaN                   NaN  \n",
       "8                        NaN                    NaN                   NaN  \n",
       "9                        NaN                    NaN                   NaN  \n",
       "10                       NaN                    NaN                   NaN  \n",
       "11                       NaN                    NaN                   NaN  \n",
       "12                       NaN                    NaN                   NaN  \n",
       "13                       NaN                    NaN                   NaN  \n",
       "14                       NaN                    NaN                   NaN  \n",
       "15                       NaN                    NaN                   NaN  \n",
       "16                       NaN                    NaN                   NaN  \n",
       "17                       NaN                    NaN                   NaN  \n",
       "18                       NaN                    NaN                   NaN  \n",
       "19                       NaN                    NaN                   NaN  \n",
       "20                       NaN                    NaN                   NaN  \n",
       "21                       NaN                    NaN                   NaN  \n",
       "22                       NaN                    NaN                   NaN  \n",
       "23                       NaN                    NaN                   NaN  \n",
       "24                       NaN                    NaN                   NaN  \n",
       "25                       NaN                    NaN                   NaN  \n",
       "26                       NaN                    NaN                   NaN  \n",
       "27                       NaN                    NaN                   NaN  \n",
       "28                       NaN                    NaN                   NaN  \n",
       "29                       NaN                    NaN                   NaN  \n",
       "...                      ...                    ...                   ...  \n",
       "289971                   NaN                    NaN                   NaN  \n",
       "289972                   NaN                    NaN                   NaN  \n",
       "289973                   NaN                    NaN                   NaN  \n",
       "289974                   NaN                    NaN                   NaN  \n",
       "289975                   NaN                    NaN                   NaN  \n",
       "289976                   NaN                    NaN                   NaN  \n",
       "289977                   NaN                    NaN                   NaN  \n",
       "289978                   NaN                    NaN                   NaN  \n",
       "289979                   NaN                    NaN                   NaN  \n",
       "289980                   NaN                    NaN                   NaN  \n",
       "289981                   NaN                    NaN                   NaN  \n",
       "289982                   NaN                    NaN                   NaN  \n",
       "289983                   NaN                    NaN                   NaN  \n",
       "289984                   NaN                    NaN                   NaN  \n",
       "289985                   NaN                    NaN                   NaN  \n",
       "289986                   NaN                    NaN                   NaN  \n",
       "289987                   NaN                    NaN                   NaN  \n",
       "289988                   NaN                    NaN                   NaN  \n",
       "289989                   NaN                    NaN                   NaN  \n",
       "289990                   NaN                    NaN                   NaN  \n",
       "289991                   NaN                    NaN                   NaN  \n",
       "289992                   NaN                    NaN                   NaN  \n",
       "289993                   NaN                    NaN                   NaN  \n",
       "289994                   NaN                    NaN                   NaN  \n",
       "289995                   NaN                    NaN                   NaN  \n",
       "289996                   NaN                    NaN                   NaN  \n",
       "289997                   NaN                    NaN                   NaN  \n",
       "289998                   NaN                    NaN                   NaN  \n",
       "289999                   NaN                    NaN                   NaN  \n",
       "290000                   NaN                    NaN                   NaN  \n",
       "\n",
       "[290001 rows x 27 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def successive_aggregates(df, field1, field2):\n",
    "    t = df.groupby(['card_id', field1])[field2].mean()\n",
    "    u = pd.DataFrame(t).reset_index().groupby('card_id')[field2].agg(['mean', 'min', 'max', 'std'])\n",
    "    u.columns = [field1 + '_' + field2 + '_' + col for col in u.columns.values]\n",
    "    u.reset_index(inplace=True)\n",
    "    return u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "y = successive_aggregates(temp_df, 'month_lag', 'purchase_amount')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>card_id</th>\n",
       "      <th>month_lag_purchase_amount_mean</th>\n",
       "      <th>month_lag_purchase_amount_min</th>\n",
       "      <th>month_lag_purchase_amount_max</th>\n",
       "      <th>month_lag_purchase_amount_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C_ID_0001238066</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C_ID_0001506ef0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C_ID_0001793786</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C_ID_000183fdda</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C_ID_0002709b5a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>C_ID_000298032a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>C_ID_0002ba3c2e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>C_ID_0002c7c2c1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>C_ID_00032df08f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>C_ID_0003754056</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>C_ID_000377f6a0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>C_ID_0003be3c83</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>C_ID_0003f41435</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>C_ID_00042d509c</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>C_ID_0004587331</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>C_ID_0004725b87</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>C_ID_0004888ddd</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>C_ID_0004b68c49</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>C_ID_0004c2a5ab</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>C_ID_00057b99fe</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>C_ID_000599daf9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>C_ID_0005f16cc8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>C_ID_0006152db8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>C_ID_000616f4a8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>C_ID_000664aa02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>C_ID_00079fec55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>C_ID_0007a60a33</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>C_ID_0007fe3157</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>C_ID_000844d8f1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289971</th>\n",
       "      <td>C_ID_fffaed41f0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289972</th>\n",
       "      <td>C_ID_fffb79fb56</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289973</th>\n",
       "      <td>C_ID_fffba72dc5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289974</th>\n",
       "      <td>C_ID_fffbdf036b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289975</th>\n",
       "      <td>C_ID_fffbee5c24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289976</th>\n",
       "      <td>C_ID_fffc96bf24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289977</th>\n",
       "      <td>C_ID_fffcb74f49</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289978</th>\n",
       "      <td>C_ID_fffcf66e12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289979</th>\n",
       "      <td>C_ID_fffd070e0e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289980</th>\n",
       "      <td>C_ID_fffd1207a8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289981</th>\n",
       "      <td>C_ID_fffd24dcf3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289982</th>\n",
       "      <td>C_ID_fffd337484</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289983</th>\n",
       "      <td>C_ID_fffd345048</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289984</th>\n",
       "      <td>C_ID_fffd351e7f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289985</th>\n",
       "      <td>C_ID_fffd93c8bf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289986</th>\n",
       "      <td>C_ID_fffd943b91</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289987</th>\n",
       "      <td>C_ID_fffdad8aa0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289988</th>\n",
       "      <td>C_ID_fffde15ab6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289989</th>\n",
       "      <td>C_ID_fffdf34798</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289990</th>\n",
       "      <td>C_ID_fffe29a04a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289991</th>\n",
       "      <td>C_ID_fffe78b232</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289992</th>\n",
       "      <td>C_ID_fffe866457</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289993</th>\n",
       "      <td>C_ID_fffea6de74</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289994</th>\n",
       "      <td>C_ID_fffeced303</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289995</th>\n",
       "      <td>C_ID_fffeed3a89</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289996</th>\n",
       "      <td>C_ID_ffff1d9928</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289997</th>\n",
       "      <td>C_ID_ffff579d3a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289998</th>\n",
       "      <td>C_ID_ffff756266</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289999</th>\n",
       "      <td>C_ID_ffff828181</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290000</th>\n",
       "      <td>C_ID_fffffd5772</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>290001 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                card_id  month_lag_purchase_amount_mean  \\\n",
       "0       C_ID_00007093c1                             NaN   \n",
       "1       C_ID_0001238066                             NaN   \n",
       "2       C_ID_0001506ef0                             NaN   \n",
       "3       C_ID_0001793786                             NaN   \n",
       "4       C_ID_000183fdda                             NaN   \n",
       "5       C_ID_0002709b5a                             NaN   \n",
       "6       C_ID_000298032a                             NaN   \n",
       "7       C_ID_0002ba3c2e                             NaN   \n",
       "8       C_ID_0002c7c2c1                             NaN   \n",
       "9       C_ID_00032df08f                             NaN   \n",
       "10      C_ID_0003754056                             NaN   \n",
       "11      C_ID_000377f6a0                             NaN   \n",
       "12      C_ID_0003be3c83                             NaN   \n",
       "13      C_ID_0003f41435                             NaN   \n",
       "14      C_ID_00042d509c                             NaN   \n",
       "15      C_ID_0004587331                             NaN   \n",
       "16      C_ID_0004725b87                             NaN   \n",
       "17      C_ID_0004888ddd                             NaN   \n",
       "18      C_ID_0004b68c49                             NaN   \n",
       "19      C_ID_0004c2a5ab                             NaN   \n",
       "20      C_ID_00057b99fe                             NaN   \n",
       "21      C_ID_000599daf9                             NaN   \n",
       "22      C_ID_0005f16cc8                             NaN   \n",
       "23      C_ID_0006152db8                             NaN   \n",
       "24      C_ID_000616f4a8                             NaN   \n",
       "25      C_ID_000664aa02                             NaN   \n",
       "26      C_ID_00079fec55                             NaN   \n",
       "27      C_ID_0007a60a33                             NaN   \n",
       "28      C_ID_0007fe3157                             NaN   \n",
       "29      C_ID_000844d8f1                             NaN   \n",
       "...                 ...                             ...   \n",
       "289971  C_ID_fffaed41f0                             NaN   \n",
       "289972  C_ID_fffb79fb56                             NaN   \n",
       "289973  C_ID_fffba72dc5                             NaN   \n",
       "289974  C_ID_fffbdf036b                             NaN   \n",
       "289975  C_ID_fffbee5c24                             NaN   \n",
       "289976  C_ID_fffc96bf24                             NaN   \n",
       "289977  C_ID_fffcb74f49                             NaN   \n",
       "289978  C_ID_fffcf66e12                             NaN   \n",
       "289979  C_ID_fffd070e0e                             NaN   \n",
       "289980  C_ID_fffd1207a8                             NaN   \n",
       "289981  C_ID_fffd24dcf3                             NaN   \n",
       "289982  C_ID_fffd337484                             NaN   \n",
       "289983  C_ID_fffd345048                             NaN   \n",
       "289984  C_ID_fffd351e7f                             NaN   \n",
       "289985  C_ID_fffd93c8bf                             NaN   \n",
       "289986  C_ID_fffd943b91                             NaN   \n",
       "289987  C_ID_fffdad8aa0                             NaN   \n",
       "289988  C_ID_fffde15ab6                             NaN   \n",
       "289989  C_ID_fffdf34798                             NaN   \n",
       "289990  C_ID_fffe29a04a                             NaN   \n",
       "289991  C_ID_fffe78b232                             NaN   \n",
       "289992  C_ID_fffe866457                             NaN   \n",
       "289993  C_ID_fffea6de74                             NaN   \n",
       "289994  C_ID_fffeced303                             NaN   \n",
       "289995  C_ID_fffeed3a89                             NaN   \n",
       "289996  C_ID_ffff1d9928                             NaN   \n",
       "289997  C_ID_ffff579d3a                             NaN   \n",
       "289998  C_ID_ffff756266                             NaN   \n",
       "289999  C_ID_ffff828181                             NaN   \n",
       "290000  C_ID_fffffd5772                             NaN   \n",
       "\n",
       "        month_lag_purchase_amount_min  month_lag_purchase_amount_max  \\\n",
       "0                                 NaN                            NaN   \n",
       "1                                 NaN                            NaN   \n",
       "2                                 NaN                            NaN   \n",
       "3                                 NaN                            NaN   \n",
       "4                                 NaN                            NaN   \n",
       "5                                 NaN                            NaN   \n",
       "6                                 NaN                            NaN   \n",
       "7                                 NaN                            NaN   \n",
       "8                                 NaN                            NaN   \n",
       "9                                 NaN                            NaN   \n",
       "10                                NaN                            NaN   \n",
       "11                                NaN                            NaN   \n",
       "12                                NaN                            NaN   \n",
       "13                                NaN                            NaN   \n",
       "14                                NaN                            NaN   \n",
       "15                                NaN                            NaN   \n",
       "16                                NaN                            NaN   \n",
       "17                                NaN                            NaN   \n",
       "18                                NaN                            NaN   \n",
       "19                                NaN                            NaN   \n",
       "20                                NaN                            NaN   \n",
       "21                                NaN                            NaN   \n",
       "22                                NaN                            NaN   \n",
       "23                                NaN                            NaN   \n",
       "24                                NaN                            NaN   \n",
       "25                                NaN                            NaN   \n",
       "26                                NaN                            NaN   \n",
       "27                                NaN                            NaN   \n",
       "28                                NaN                            NaN   \n",
       "29                                NaN                            NaN   \n",
       "...                               ...                            ...   \n",
       "289971                            NaN                            NaN   \n",
       "289972                            NaN                            NaN   \n",
       "289973                            NaN                            NaN   \n",
       "289974                            NaN                            NaN   \n",
       "289975                            NaN                            NaN   \n",
       "289976                            NaN                            NaN   \n",
       "289977                            NaN                            NaN   \n",
       "289978                            NaN                            NaN   \n",
       "289979                            NaN                            NaN   \n",
       "289980                            NaN                            NaN   \n",
       "289981                            NaN                            NaN   \n",
       "289982                            NaN                            NaN   \n",
       "289983                            NaN                            NaN   \n",
       "289984                            NaN                            NaN   \n",
       "289985                            NaN                            NaN   \n",
       "289986                            NaN                            NaN   \n",
       "289987                            NaN                            NaN   \n",
       "289988                            NaN                            NaN   \n",
       "289989                            NaN                            NaN   \n",
       "289990                            NaN                            NaN   \n",
       "289991                            NaN                            NaN   \n",
       "289992                            NaN                            NaN   \n",
       "289993                            NaN                            NaN   \n",
       "289994                            NaN                            NaN   \n",
       "289995                            NaN                            NaN   \n",
       "289996                            NaN                            NaN   \n",
       "289997                            NaN                            NaN   \n",
       "289998                            NaN                            NaN   \n",
       "289999                            NaN                            NaN   \n",
       "290000                            NaN                            NaN   \n",
       "\n",
       "        month_lag_purchase_amount_std  \n",
       "0                                 NaN  \n",
       "1                                 NaN  \n",
       "2                                 NaN  \n",
       "3                                 NaN  \n",
       "4                                 NaN  \n",
       "5                                 NaN  \n",
       "6                                 NaN  \n",
       "7                                 NaN  \n",
       "8                                 NaN  \n",
       "9                                 NaN  \n",
       "10                                NaN  \n",
       "11                                NaN  \n",
       "12                                NaN  \n",
       "13                                NaN  \n",
       "14                                NaN  \n",
       "15                                NaN  \n",
       "16                                NaN  \n",
       "17                                NaN  \n",
       "18                                NaN  \n",
       "19                                NaN  \n",
       "20                                NaN  \n",
       "21                                NaN  \n",
       "22                                NaN  \n",
       "23                                NaN  \n",
       "24                                NaN  \n",
       "25                                NaN  \n",
       "26                                NaN  \n",
       "27                                NaN  \n",
       "28                                NaN  \n",
       "29                                NaN  \n",
       "...                               ...  \n",
       "289971                            NaN  \n",
       "289972                            NaN  \n",
       "289973                            NaN  \n",
       "289974                            NaN  \n",
       "289975                            NaN  \n",
       "289976                            NaN  \n",
       "289977                            NaN  \n",
       "289978                            NaN  \n",
       "289979                            NaN  \n",
       "289980                            NaN  \n",
       "289981                            NaN  \n",
       "289982                            NaN  \n",
       "289983                            NaN  \n",
       "289984                            NaN  \n",
       "289985                            NaN  \n",
       "289986                            NaN  \n",
       "289987                            NaN  \n",
       "289988                            NaN  \n",
       "289989                            NaN  \n",
       "289990                            NaN  \n",
       "289991                            NaN  \n",
       "289992                            NaN  \n",
       "289993                            NaN  \n",
       "289994                            NaN  \n",
       "289995                            NaN  \n",
       "289996                            NaN  \n",
       "289997                            NaN  \n",
       "289998                            NaN  \n",
       "289999                            NaN  \n",
       "290000                            NaN  \n",
       "\n",
       "[290001 rows x 5 columns]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "temp_df.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/frame.py:3697: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n"
     ]
    }
   ],
   "source": [
    "temp_df.drop('index', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorized_flag</th>\n",
       "      <th>card_id</th>\n",
       "      <th>city_id</th>\n",
       "      <th>category_1</th>\n",
       "      <th>installments</th>\n",
       "      <th>category_3</th>\n",
       "      <th>merchant_category_id</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th>purchase_amount</th>\n",
       "      <th>...</th>\n",
       "      <th>purchase_Dayofyear</th>\n",
       "      <th>purchase_Is_month_end</th>\n",
       "      <th>purchase_Is_month_start</th>\n",
       "      <th>purchase_Is_quarter_end</th>\n",
       "      <th>purchase_Is_quarter_start</th>\n",
       "      <th>purchase_Is_year_end</th>\n",
       "      <th>purchase_Is_year_start</th>\n",
       "      <th>purchase_Elapsed</th>\n",
       "      <th>purchased_on_weekend</th>\n",
       "      <th>month_diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>231</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>278</td>\n",
       "      <td>M_ID_2ae6547567</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.654885</td>\n",
       "      <td>...</td>\n",
       "      <td>79</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1521542253</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>C</td>\n",
       "      <td>491</td>\n",
       "      <td>M_ID_79692349d6</td>\n",
       "      <td>1</td>\n",
       "      <td>3.009720</td>\n",
       "      <td>...</td>\n",
       "      <td>89</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1522440354</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>231</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>360</td>\n",
       "      <td>M_ID_532b2abc39</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.432253</td>\n",
       "      <td>...</td>\n",
       "      <td>96</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1523021302</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>C</td>\n",
       "      <td>87</td>\n",
       "      <td>M_ID_99c89d936e</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.357721</td>\n",
       "      <td>...</td>\n",
       "      <td>83</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1521903510</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>C</td>\n",
       "      <td>690</td>\n",
       "      <td>M_ID_3111c6df35</td>\n",
       "      <td>1</td>\n",
       "      <td>2.126146</td>\n",
       "      <td>...</td>\n",
       "      <td>62</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1520077258</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>195</td>\n",
       "      <td>M_ID_7fc4685211</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.704834</td>\n",
       "      <td>...</td>\n",
       "      <td>76</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1521289951</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_241a01e9d9</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>C</td>\n",
       "      <td>884</td>\n",
       "      <td>M_ID_7c26306ee3</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.461705</td>\n",
       "      <td>...</td>\n",
       "      <td>112</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1524424748</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_241a01e9d9</td>\n",
       "      <td>188</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>C</td>\n",
       "      <td>690</td>\n",
       "      <td>M_ID_7c7d14fd19</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.294760</td>\n",
       "      <td>...</td>\n",
       "      <td>112</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1524425651</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_a97720321f</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "      <td>419</td>\n",
       "      <td>M_ID_7064a8aa68</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.202377</td>\n",
       "      <td>...</td>\n",
       "      <td>61</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1519992383</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_a97720321f</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "      <td>705</td>\n",
       "      <td>M_ID_1f4773aa76</td>\n",
       "      <td>1</td>\n",
       "      <td>0.755743</td>\n",
       "      <td>...</td>\n",
       "      <td>62</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1520079298</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   authorized_flag          card_id city_id  category_1  installments  \\\n",
       "0                1  C_ID_ef55cf8d4b     231           0             1   \n",
       "1                1  C_ID_ef55cf8d4b      -1           1             3   \n",
       "2                1  C_ID_ef55cf8d4b     231           0             1   \n",
       "3                1  C_ID_ef55cf8d4b      69           0             2   \n",
       "4                1  C_ID_ef55cf8d4b      -1           1            12   \n",
       "5                1  C_ID_ef55cf8d4b      69           0             1   \n",
       "6                1  C_ID_241a01e9d9      69           0             3   \n",
       "7                1  C_ID_241a01e9d9     188           0             6   \n",
       "8                1  C_ID_a97720321f      69           0             0   \n",
       "9                1  C_ID_a97720321f      69           0             0   \n",
       "\n",
       "  category_3 merchant_category_id      merchant_id  month_lag  \\\n",
       "0          B                  278  M_ID_2ae6547567          1   \n",
       "1          C                  491  M_ID_79692349d6          1   \n",
       "2          B                  360  M_ID_532b2abc39          2   \n",
       "3          C                   87  M_ID_99c89d936e          1   \n",
       "4          C                  690  M_ID_3111c6df35          1   \n",
       "5          B                  195  M_ID_7fc4685211          1   \n",
       "6          C                  884  M_ID_7c26306ee3          2   \n",
       "7          C                  690  M_ID_7c7d14fd19          2   \n",
       "8          A                  419  M_ID_7064a8aa68          1   \n",
       "9          A                  705  M_ID_1f4773aa76          1   \n",
       "\n",
       "   purchase_amount     ...     purchase_Dayofyear purchase_Is_month_end  \\\n",
       "0        -0.654885     ...                     79                 False   \n",
       "1         3.009720     ...                     89                 False   \n",
       "2        -0.432253     ...                     96                 False   \n",
       "3        -0.357721     ...                     83                 False   \n",
       "4         2.126146     ...                     62                 False   \n",
       "5        -0.704834     ...                     76                 False   \n",
       "6        -0.461705     ...                    112                 False   \n",
       "7        -0.294760     ...                    112                 False   \n",
       "8        -0.202377     ...                     61                 False   \n",
       "9         0.755743     ...                     62                 False   \n",
       "\n",
       "  purchase_Is_month_start purchase_Is_quarter_end purchase_Is_quarter_start  \\\n",
       "0                   False                   False                     False   \n",
       "1                   False                   False                     False   \n",
       "2                   False                   False                     False   \n",
       "3                   False                   False                     False   \n",
       "4                   False                   False                     False   \n",
       "5                   False                   False                     False   \n",
       "6                   False                   False                     False   \n",
       "7                   False                   False                     False   \n",
       "8                   False                   False                     False   \n",
       "9                   False                   False                     False   \n",
       "\n",
       "  purchase_Is_year_end  purchase_Is_year_start  purchase_Elapsed  \\\n",
       "0                False                   False        1521542253   \n",
       "1                False                   False        1522440354   \n",
       "2                False                   False        1523021302   \n",
       "3                False                   False        1521903510   \n",
       "4                False                   False        1520077258   \n",
       "5                False                   False        1521289951   \n",
       "6                False                   False        1524424748   \n",
       "7                False                   False        1524425651   \n",
       "8                False                   False        1519992383   \n",
       "9                False                   False        1520079298   \n",
       "\n",
       "   purchased_on_weekend  month_diff  \n",
       "0                     0          10  \n",
       "1                     0          10  \n",
       "2                     0          11  \n",
       "3                     1          10  \n",
       "4                     1          11  \n",
       "5                     1          10  \n",
       "6                     1          10  \n",
       "7                     1          10  \n",
       "8                     0          11  \n",
       "9                     1          11  \n",
       "\n",
       "[10 rows x 29 columns]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load train & test csvs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = 'data/elo/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = [pd.read_csv(f'{PATH}{c}') for c in ['train.csv', 'test.csv']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>first_active_month</th>\n",
       "      <td>2017-06</td>\n",
       "      <td>2017-01</td>\n",
       "      <td>2016-08</td>\n",
       "      <td>2017-09</td>\n",
       "      <td>2017-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <td>C_ID_92a2005557</td>\n",
       "      <td>C_ID_3d0044924f</td>\n",
       "      <td>C_ID_d639edf6cd</td>\n",
       "      <td>C_ID_186d6a6901</td>\n",
       "      <td>C_ID_cdbd2c0db2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>-0.820283</td>\n",
       "      <td>0.392913</td>\n",
       "      <td>0.688056</td>\n",
       "      <td>0.142495</td>\n",
       "      <td>-0.159749</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  0                1                2  \\\n",
       "first_active_month          2017-06          2017-01          2016-08   \n",
       "card_id             C_ID_92a2005557  C_ID_3d0044924f  C_ID_d639edf6cd   \n",
       "feature_1                         5                4                2   \n",
       "feature_2                         2                1                2   \n",
       "feature_3                         1                0                0   \n",
       "target                    -0.820283         0.392913         0.688056   \n",
       "\n",
       "                                  3                4  \n",
       "first_active_month          2017-09          2017-11  \n",
       "card_id             C_ID_186d6a6901  C_ID_cdbd2c0db2  \n",
       "feature_1                         4                1  \n",
       "feature_2                         3                3  \n",
       "feature_3                         0                0  \n",
       "target                     0.142495        -0.159749  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "add_datepart(train, 'first_active_month', drop=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>first_active_month</th>\n",
       "      <td>2017-06-01 00:00:00</td>\n",
       "      <td>2017-01-01 00:00:00</td>\n",
       "      <td>2016-08-01 00:00:00</td>\n",
       "      <td>2017-09-01 00:00:00</td>\n",
       "      <td>2017-11-01 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <td>C_ID_92a2005557</td>\n",
       "      <td>C_ID_3d0044924f</td>\n",
       "      <td>C_ID_d639edf6cd</td>\n",
       "      <td>C_ID_186d6a6901</td>\n",
       "      <td>C_ID_cdbd2c0db2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>-0.820283</td>\n",
       "      <td>0.392913</td>\n",
       "      <td>0.688056</td>\n",
       "      <td>0.142495</td>\n",
       "      <td>-0.159749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthYear</th>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "      <td>2016</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthMonth</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthWeek</th>\n",
       "      <td>22</td>\n",
       "      <td>52</td>\n",
       "      <td>31</td>\n",
       "      <td>35</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDay</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDayofweek</th>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDayofyear</th>\n",
       "      <td>152</td>\n",
       "      <td>1</td>\n",
       "      <td>214</td>\n",
       "      <td>244</td>\n",
       "      <td>305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_month_end</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_month_start</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_quarter_end</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_quarter_start</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_year_end</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_year_start</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthElapsed</th>\n",
       "      <td>1496275200</td>\n",
       "      <td>1483228800</td>\n",
       "      <td>1470009600</td>\n",
       "      <td>1504224000</td>\n",
       "      <td>1509494400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      0                    1  \\\n",
       "first_active_month                  2017-06-01 00:00:00  2017-01-01 00:00:00   \n",
       "card_id                                 C_ID_92a2005557      C_ID_3d0044924f   \n",
       "feature_1                                             5                    4   \n",
       "feature_2                                             2                    1   \n",
       "feature_3                                             1                    0   \n",
       "target                                        -0.820283             0.392913   \n",
       "first_active_monthYear                             2017                 2017   \n",
       "first_active_monthMonth                               6                    1   \n",
       "first_active_monthWeek                               22                   52   \n",
       "first_active_monthDay                                 1                    1   \n",
       "first_active_monthDayofweek                           3                    6   \n",
       "first_active_monthDayofyear                         152                    1   \n",
       "first_active_monthIs_month_end                    False                False   \n",
       "first_active_monthIs_month_start                   True                 True   \n",
       "first_active_monthIs_quarter_end                  False                False   \n",
       "first_active_monthIs_quarter_start                False                 True   \n",
       "first_active_monthIs_year_end                     False                False   \n",
       "first_active_monthIs_year_start                   False                 True   \n",
       "first_active_monthElapsed                    1496275200           1483228800   \n",
       "\n",
       "                                                      2                    3  \\\n",
       "first_active_month                  2016-08-01 00:00:00  2017-09-01 00:00:00   \n",
       "card_id                                 C_ID_d639edf6cd      C_ID_186d6a6901   \n",
       "feature_1                                             2                    4   \n",
       "feature_2                                             2                    3   \n",
       "feature_3                                             0                    0   \n",
       "target                                         0.688056             0.142495   \n",
       "first_active_monthYear                             2016                 2017   \n",
       "first_active_monthMonth                               8                    9   \n",
       "first_active_monthWeek                               31                   35   \n",
       "first_active_monthDay                                 1                    1   \n",
       "first_active_monthDayofweek                           0                    4   \n",
       "first_active_monthDayofyear                         214                  244   \n",
       "first_active_monthIs_month_end                    False                False   \n",
       "first_active_monthIs_month_start                   True                 True   \n",
       "first_active_monthIs_quarter_end                  False                False   \n",
       "first_active_monthIs_quarter_start                False                False   \n",
       "first_active_monthIs_year_end                     False                False   \n",
       "first_active_monthIs_year_start                   False                False   \n",
       "first_active_monthElapsed                    1470009600           1504224000   \n",
       "\n",
       "                                                      4  \n",
       "first_active_month                  2017-11-01 00:00:00  \n",
       "card_id                                 C_ID_cdbd2c0db2  \n",
       "feature_1                                             1  \n",
       "feature_2                                             3  \n",
       "feature_3                                             0  \n",
       "target                                        -0.159749  \n",
       "first_active_monthYear                             2017  \n",
       "first_active_monthMonth                              11  \n",
       "first_active_monthWeek                               44  \n",
       "first_active_monthDay                                 1  \n",
       "first_active_monthDayofweek                           2  \n",
       "first_active_monthDayofyear                         305  \n",
       "first_active_monthIs_month_end                    False  \n",
       "first_active_monthIs_month_start                   True  \n",
       "first_active_monthIs_quarter_end                  False  \n",
       "first_active_monthIs_quarter_start                False  \n",
       "first_active_monthIs_year_end                     False  \n",
       "first_active_monthIs_year_start                   False  \n",
       "first_active_monthElapsed                    1509494400  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "add_datepart(test, 'first_active_month', drop=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist_trans = pd.read_csv(f'{PATH}historical_transactions.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fill_nas_for_transactions_df(df):\n",
    "    # Fill nas for category_3 with mode\n",
    "    cols = ['merchant_id', 'category_3', 'category_2']\n",
    "    for col in cols:\n",
    "        df[col] = df[col].fillna(df[col].mode()[0])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "hist_trans = fill_nas_for_transactions_df(hist_trans[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>authorized_flag</th>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <td>C_ID_4e6213e9bc</td>\n",
       "      <td>C_ID_4e6213e9bc</td>\n",
       "      <td>C_ID_4e6213e9bc</td>\n",
       "      <td>C_ID_4e6213e9bc</td>\n",
       "      <td>C_ID_4e6213e9bc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city_id</th>\n",
       "      <td>88</td>\n",
       "      <td>88</td>\n",
       "      <td>88</td>\n",
       "      <td>88</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_1</th>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_3</th>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_category_id</th>\n",
       "      <td>80</td>\n",
       "      <td>367</td>\n",
       "      <td>80</td>\n",
       "      <td>560</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_id</th>\n",
       "      <td>M_ID_e020e9b302</td>\n",
       "      <td>M_ID_86ec983688</td>\n",
       "      <td>M_ID_979ed661fc</td>\n",
       "      <td>M_ID_e6d5ae8ea6</td>\n",
       "      <td>M_ID_e020e9b302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag</th>\n",
       "      <td>-8</td>\n",
       "      <td>-7</td>\n",
       "      <td>-6</td>\n",
       "      <td>-5</td>\n",
       "      <td>-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount</th>\n",
       "      <td>-0.703331</td>\n",
       "      <td>-0.733128</td>\n",
       "      <td>-0.720386</td>\n",
       "      <td>-0.735352</td>\n",
       "      <td>-0.722865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date</th>\n",
       "      <td>2017-06-25 15:33:07</td>\n",
       "      <td>2017-07-15 12:10:45</td>\n",
       "      <td>2017-08-09 22:04:29</td>\n",
       "      <td>2017-09-02 10:06:26</td>\n",
       "      <td>2017-03-10 01:14:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_id</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsector_id</th>\n",
       "      <td>37</td>\n",
       "      <td>16</td>\n",
       "      <td>37</td>\n",
       "      <td>34</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        0                    1  \\\n",
       "authorized_flag                         Y                    Y   \n",
       "card_id                   C_ID_4e6213e9bc      C_ID_4e6213e9bc   \n",
       "city_id                                88                   88   \n",
       "category_1                              N                    N   \n",
       "installments                            0                    0   \n",
       "category_3                              A                    A   \n",
       "merchant_category_id                   80                  367   \n",
       "merchant_id               M_ID_e020e9b302      M_ID_86ec983688   \n",
       "month_lag                              -8                   -7   \n",
       "purchase_amount                 -0.703331            -0.733128   \n",
       "purchase_date         2017-06-25 15:33:07  2017-07-15 12:10:45   \n",
       "category_2                              1                    1   \n",
       "state_id                               16                   16   \n",
       "subsector_id                           37                   16   \n",
       "\n",
       "                                        2                    3  \\\n",
       "authorized_flag                         Y                    Y   \n",
       "card_id                   C_ID_4e6213e9bc      C_ID_4e6213e9bc   \n",
       "city_id                                88                   88   \n",
       "category_1                              N                    N   \n",
       "installments                            0                    0   \n",
       "category_3                              A                    A   \n",
       "merchant_category_id                   80                  560   \n",
       "merchant_id               M_ID_979ed661fc      M_ID_e6d5ae8ea6   \n",
       "month_lag                              -6                   -5   \n",
       "purchase_amount                 -0.720386            -0.735352   \n",
       "purchase_date         2017-08-09 22:04:29  2017-09-02 10:06:26   \n",
       "category_2                              1                    1   \n",
       "state_id                               16                   16   \n",
       "subsector_id                           37                   34   \n",
       "\n",
       "                                        4  \n",
       "authorized_flag                         Y  \n",
       "card_id                   C_ID_4e6213e9bc  \n",
       "city_id                                88  \n",
       "category_1                              N  \n",
       "installments                            0  \n",
       "category_3                              A  \n",
       "merchant_category_id                   80  \n",
       "merchant_id               M_ID_e020e9b302  \n",
       "month_lag                             -11  \n",
       "purchase_amount                 -0.722865  \n",
       "purchase_date         2017-03-10 01:14:19  \n",
       "category_2                              1  \n",
       "state_id                               16  \n",
       "subsector_id                           37  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hist_trans.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "add_datepart(hist_trans, 'purchase_date', drop=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 27)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hist_trans.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1963031, 29)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_merchant_transactions_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist_trans = add_extra_cols(hist_trans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 29)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hist_trans.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist_trans = mark_categorical_cols_for_transactions(hist_trans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist_trans.to_feather('hist_trans')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist_trans = feather.read_dataframe('hist_trans')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 24.2 ms, sys: 4.01 ms, total: 28.2 ms\n",
      "Wall time: 27.7 ms\n"
     ]
    }
   ],
   "source": [
    "%time hist_trans_agg = aggregate_by_card_id(hist_trans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "authorized_flag                       int64\n",
       "card_id                            category\n",
       "city_id                            category\n",
       "category_1                            int64\n",
       "installments                          int64\n",
       "category_3                           object\n",
       "merchant_category_id               category\n",
       "merchant_id                        category\n",
       "month_lag                             int64\n",
       "purchase_amount                     float64\n",
       "purchase_date                datetime64[ns]\n",
       "category_2                          float64\n",
       "state_id                           category\n",
       "subsector_id                       category\n",
       "purchase_Year                      category\n",
       "purchase_Month                     category\n",
       "purchase_Week                         int64\n",
       "purchase_Day                          int64\n",
       "purchase_Dayofweek                    int64\n",
       "purchase_Dayofyear                    int64\n",
       "purchase_Is_month_end                  bool\n",
       "purchase_Is_month_start                bool\n",
       "purchase_Is_quarter_end                bool\n",
       "purchase_Is_quarter_start              bool\n",
       "purchase_Is_year_end                   bool\n",
       "purchase_Is_year_start                 bool\n",
       "purchase_Elapsed                      int64\n",
       "purchased_on_weekend                  int64\n",
       "month_diff                            int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hist_trans.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(325540, 37)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hist_trans_agg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(290001, 39)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_hist_agg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist_trans_agg = add_extra_cols_on_agg(hist_trans_agg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(325540, 39)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hist_trans_agg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist_trans_agg.to_feather('hist_trans_agg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.merge(train, new_hist_agg, how='left', on='card_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "      <th>counts</th>\n",
       "      <th>uniques</th>\n",
       "      <th>missing</th>\n",
       "      <th>missing_perc</th>\n",
       "      <th>types</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>first_active_month</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>unique</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1</th>\n",
       "      <td>201917</td>\n",
       "      <td>3.10531</td>\n",
       "      <td>1.18616</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>201917</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2</th>\n",
       "      <td>201917</td>\n",
       "      <td>1.74541</td>\n",
       "      <td>0.751362</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>201917</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_3</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.565569</td>\n",
       "      <td>0.495683</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>201917</td>\n",
       "      <td>-0.393636</td>\n",
       "      <td>3.8505</td>\n",
       "      <td>-33.2193</td>\n",
       "      <td>-0.88311</td>\n",
       "      <td>-0.0234369</td>\n",
       "      <td>0.765453</td>\n",
       "      <td>17.9651</td>\n",
       "      <td>201917</td>\n",
       "      <td>197110</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthYear</th>\n",
       "      <td>201917</td>\n",
       "      <td>2016.51</td>\n",
       "      <td>0.788199</td>\n",
       "      <td>2011</td>\n",
       "      <td>2016</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "      <td>2018</td>\n",
       "      <td>201917</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthMonth</th>\n",
       "      <td>201917</td>\n",
       "      <td>7.37874</td>\n",
       "      <td>3.34072</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>201917</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthWeek</th>\n",
       "      <td>201917</td>\n",
       "      <td>30.9446</td>\n",
       "      <td>13.8454</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>31</td>\n",
       "      <td>44</td>\n",
       "      <td>53</td>\n",
       "      <td>201917</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDay</th>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDayofweek</th>\n",
       "      <td>201917</td>\n",
       "      <td>3.17948</td>\n",
       "      <td>1.89371</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>201917</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDayofyear</th>\n",
       "      <td>201917</td>\n",
       "      <td>194.482</td>\n",
       "      <td>101.796</td>\n",
       "      <td>1</td>\n",
       "      <td>121</td>\n",
       "      <td>213</td>\n",
       "      <td>275</td>\n",
       "      <td>336</td>\n",
       "      <td>201917</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_month_end</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_month_start</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_quarter_end</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_quarter_start</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_year_end</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_year_start</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthElapsed</th>\n",
       "      <td>201917</td>\n",
       "      <td>1.48444e+09</td>\n",
       "      <td>2.53766e+07</td>\n",
       "      <td>1.32011e+09</td>\n",
       "      <td>1.47528e+09</td>\n",
       "      <td>1.491e+09</td>\n",
       "      <td>1.50422e+09</td>\n",
       "      <td>1.51744e+09</td>\n",
       "      <td>201917</td>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transactions_count</th>\n",
       "      <td>179986</td>\n",
       "      <td>6.77655</td>\n",
       "      <td>6.69458</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>109</td>\n",
       "      <td>179986</td>\n",
       "      <td>87</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city_id_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>2.30943</td>\n",
       "      <td>1.67011</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>28</td>\n",
       "      <td>179986</td>\n",
       "      <td>26</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_category_id_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>5.33923</td>\n",
       "      <td>4.25534</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>40</td>\n",
       "      <td>179986</td>\n",
       "      <td>39</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_id_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>6.68335</td>\n",
       "      <td>6.65009</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>109</td>\n",
       "      <td>179986</td>\n",
       "      <td>87</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_id_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>1.58205</td>\n",
       "      <td>0.873557</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>179986</td>\n",
       "      <td>13</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Month_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>1.65671</td>\n",
       "      <td>0.47481</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>179986</td>\n",
       "      <td>2</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsector_id_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>4.45392</td>\n",
       "      <td>3.05844</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>179986</td>\n",
       "      <td>24</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>201917</td>\n",
       "      <td>100%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_sum</th>\n",
       "      <td>179986</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>179986</td>\n",
       "      <td>1</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_sum</th>\n",
       "      <td>179986</td>\n",
       "      <td>-3.74189</td>\n",
       "      <td>4.39531</td>\n",
       "      <td>-70.8135</td>\n",
       "      <td>-5.25733</td>\n",
       "      <td>-2.68625</td>\n",
       "      <td>-1.21177</td>\n",
       "      <td>109.086</td>\n",
       "      <td>179986</td>\n",
       "      <td>149495</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_max</th>\n",
       "      <td>179986</td>\n",
       "      <td>-0.131691</td>\n",
       "      <td>1.53213</td>\n",
       "      <td>-0.746893</td>\n",
       "      <td>-0.647883</td>\n",
       "      <td>-0.51302</td>\n",
       "      <td>-0.181911</td>\n",
       "      <td>76.6829</td>\n",
       "      <td>179986</td>\n",
       "      <td>43853</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_min</th>\n",
       "      <td>179986</td>\n",
       "      <td>-0.660582</td>\n",
       "      <td>0.433409</td>\n",
       "      <td>-0.746893</td>\n",
       "      <td>-0.735638</td>\n",
       "      <td>-0.723466</td>\n",
       "      <td>-0.692076</td>\n",
       "      <td>44.1824</td>\n",
       "      <td>179986</td>\n",
       "      <td>15978</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_mean</th>\n",
       "      <td>179986</td>\n",
       "      <td>-0.524392</td>\n",
       "      <td>0.554886</td>\n",
       "      <td>-0.746893</td>\n",
       "      <td>-0.692782</td>\n",
       "      <td>-0.641737</td>\n",
       "      <td>-0.531653</td>\n",
       "      <td>44.1824</td>\n",
       "      <td>179986</td>\n",
       "      <td>146245</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_std</th>\n",
       "      <td>153199</td>\n",
       "      <td>0.231553</td>\n",
       "      <td>0.592297</td>\n",
       "      <td>0</td>\n",
       "      <td>0.039286</td>\n",
       "      <td>0.0890018</td>\n",
       "      <td>0.212109</td>\n",
       "      <td>47.3277</td>\n",
       "      <td>153199</td>\n",
       "      <td>145877</td>\n",
       "      <td>48718</td>\n",
       "      <td>24.13%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_var</th>\n",
       "      <td>153199</td>\n",
       "      <td>0.40443</td>\n",
       "      <td>9.48997</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00154339</td>\n",
       "      <td>0.00792133</td>\n",
       "      <td>0.04499</td>\n",
       "      <td>2239.92</td>\n",
       "      <td>153199</td>\n",
       "      <td>146001</td>\n",
       "      <td>48718</td>\n",
       "      <td>24.13%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_sum</th>\n",
       "      <td>179986</td>\n",
       "      <td>4.62656</td>\n",
       "      <td>8.19539</td>\n",
       "      <td>-6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1004</td>\n",
       "      <td>179986</td>\n",
       "      <td>107</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_max</th>\n",
       "      <td>179986</td>\n",
       "      <td>1.47975</td>\n",
       "      <td>3.35991</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>999</td>\n",
       "      <td>179986</td>\n",
       "      <td>15</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_min</th>\n",
       "      <td>179986</td>\n",
       "      <td>0.328637</td>\n",
       "      <td>1.03692</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>179986</td>\n",
       "      <td>14</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_mean</th>\n",
       "      <td>179986</td>\n",
       "      <td>0.744392</td>\n",
       "      <td>1.13814</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1</td>\n",
       "      <td>34.6207</td>\n",
       "      <td>179986</td>\n",
       "      <td>1104</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_std</th>\n",
       "      <td>153199</td>\n",
       "      <td>0.519733</td>\n",
       "      <td>1.09463</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.755929</td>\n",
       "      <td>185.477</td>\n",
       "      <td>153199</td>\n",
       "      <td>7102</td>\n",
       "      <td>48718</td>\n",
       "      <td>24.13%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_var</th>\n",
       "      <td>153199</td>\n",
       "      <td>1.46832</td>\n",
       "      <td>87.9967</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>34401.7</td>\n",
       "      <td>153199</td>\n",
       "      <td>8167</td>\n",
       "      <td>48718</td>\n",
       "      <td>24.13%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_sum</th>\n",
       "      <td>179986</td>\n",
       "      <td>10.0025</td>\n",
       "      <td>10.0975</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>185</td>\n",
       "      <td>179986</td>\n",
       "      <td>126</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_max</th>\n",
       "      <td>179986</td>\n",
       "      <td>1.81539</td>\n",
       "      <td>0.387985</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>179986</td>\n",
       "      <td>2</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_min</th>\n",
       "      <td>179986</td>\n",
       "      <td>1.15868</td>\n",
       "      <td>0.365378</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>179986</td>\n",
       "      <td>2</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_mean</th>\n",
       "      <td>179986</td>\n",
       "      <td>1.47622</td>\n",
       "      <td>0.3308</td>\n",
       "      <td>1</td>\n",
       "      <td>1.23077</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1.71429</td>\n",
       "      <td>2</td>\n",
       "      <td>179986</td>\n",
       "      <td>690</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_std</th>\n",
       "      <td>153199</td>\n",
       "      <td>0.396778</td>\n",
       "      <td>0.227721</td>\n",
       "      <td>0</td>\n",
       "      <td>0.351091</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.534522</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>153199</td>\n",
       "      <td>1473</td>\n",
       "      <td>48718</td>\n",
       "      <td>24.13%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_var</th>\n",
       "      <td>153199</td>\n",
       "      <td>0.209289</td>\n",
       "      <td>0.137089</td>\n",
       "      <td>0</td>\n",
       "      <td>0.123265</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.5</td>\n",
       "      <td>153199</td>\n",
       "      <td>1946</td>\n",
       "      <td>48718</td>\n",
       "      <td>24.13%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amin</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>179986</td>\n",
       "      <td>172918</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amax</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>179986</td>\n",
       "      <td>173711</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_ptp</th>\n",
       "      <td>179986</td>\n",
       "      <td>29.0291</td>\n",
       "      <td>20.0683</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>32</td>\n",
       "      <td>47</td>\n",
       "      <td>61</td>\n",
       "      <td>179986</td>\n",
       "      <td>62</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id_size</th>\n",
       "      <td>179986</td>\n",
       "      <td>6.77655</td>\n",
       "      <td>6.69458</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>109</td>\n",
       "      <td>179986</td>\n",
       "      <td>87</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_sum</th>\n",
       "      <td>179986</td>\n",
       "      <td>2.0832</td>\n",
       "      <td>2.72596</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>45</td>\n",
       "      <td>179986</td>\n",
       "      <td>39</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_mean</th>\n",
       "      <td>179986</td>\n",
       "      <td>0.292901</td>\n",
       "      <td>0.287023</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1</td>\n",
       "      <td>179986</td>\n",
       "      <td>579</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff_mean</th>\n",
       "      <td>179986</td>\n",
       "      <td>11.668</td>\n",
       "      <td>2.41644</td>\n",
       "      <td>10</td>\n",
       "      <td>10.4</td>\n",
       "      <td>10.7143</td>\n",
       "      <td>11.6</td>\n",
       "      <td>23</td>\n",
       "      <td>179986</td>\n",
       "      <td>2790</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_1_sum</th>\n",
       "      <td>179986</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>179986</td>\n",
       "      <td>1</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_1_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>201917</td>\n",
       "      <td>100%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inverse_avg_transactions_per_day</th>\n",
       "      <td>179986</td>\n",
       "      <td>5.01168</td>\n",
       "      <td>4.55844</td>\n",
       "      <td>0</td>\n",
       "      <td>1.70588</td>\n",
       "      <td>4.14286</td>\n",
       "      <td>7.125</td>\n",
       "      <td>30</td>\n",
       "      <td>179986</td>\n",
       "      <td>1292</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_since_last_transaction</th>\n",
       "      <td>179986</td>\n",
       "      <td>305.242</td>\n",
       "      <td>73.4597</td>\n",
       "      <td>255</td>\n",
       "      <td>261</td>\n",
       "      <td>276</td>\n",
       "      <td>310</td>\n",
       "      <td>678</td>\n",
       "      <td>179986</td>\n",
       "      <td>406</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     count         mean          std  \\\n",
       "first_active_month                     NaN          NaN          NaN   \n",
       "card_id                                NaN          NaN          NaN   \n",
       "feature_1                           201917      3.10531      1.18616   \n",
       "feature_2                           201917      1.74541     0.751362   \n",
       "feature_3                           201917     0.565569     0.495683   \n",
       "target                              201917    -0.393636       3.8505   \n",
       "first_active_monthYear              201917      2016.51     0.788199   \n",
       "first_active_monthMonth             201917      7.37874      3.34072   \n",
       "first_active_monthWeek              201917      30.9446      13.8454   \n",
       "first_active_monthDay               201917            1            0   \n",
       "first_active_monthDayofweek         201917      3.17948      1.89371   \n",
       "first_active_monthDayofyear         201917      194.482      101.796   \n",
       "first_active_monthIs_month_end         NaN          NaN          NaN   \n",
       "first_active_monthIs_month_start       NaN          NaN          NaN   \n",
       "first_active_monthIs_quarter_end       NaN          NaN          NaN   \n",
       "first_active_monthIs_quarter_start     NaN          NaN          NaN   \n",
       "first_active_monthIs_year_end          NaN          NaN          NaN   \n",
       "first_active_monthIs_year_start        NaN          NaN          NaN   \n",
       "first_active_monthElapsed           201917  1.48444e+09  2.53766e+07   \n",
       "transactions_count                  179986      6.77655      6.69458   \n",
       "city_id_nunique                     179986      2.30943      1.67011   \n",
       "merchant_category_id_nunique        179986      5.33923      4.25534   \n",
       "merchant_id_nunique                 179986      6.68335      6.65009   \n",
       "state_id_nunique                    179986      1.58205     0.873557   \n",
       "purchase_Month_nunique              179986      1.65671      0.47481   \n",
       "subsector_id_nunique                179986      4.45392      3.05844   \n",
       "authorized_flag_mean                     0          NaN          NaN   \n",
       "authorized_flag_sum                 179986            0            0   \n",
       "purchase_amount_sum                 179986     -3.74189      4.39531   \n",
       "purchase_amount_max                 179986    -0.131691      1.53213   \n",
       "purchase_amount_min                 179986    -0.660582     0.433409   \n",
       "purchase_amount_mean                179986    -0.524392     0.554886   \n",
       "purchase_amount_std                 153199     0.231553     0.592297   \n",
       "purchase_amount_var                 153199      0.40443      9.48997   \n",
       "installments_sum                    179986      4.62656      8.19539   \n",
       "installments_max                    179986      1.47975      3.35991   \n",
       "installments_min                    179986     0.328637      1.03692   \n",
       "installments_mean                   179986     0.744392      1.13814   \n",
       "installments_std                    153199     0.519733      1.09463   \n",
       "installments_var                    153199      1.46832      87.9967   \n",
       "month_lag_sum                       179986      10.0025      10.0975   \n",
       "month_lag_max                       179986      1.81539     0.387985   \n",
       "month_lag_min                       179986      1.15868     0.365378   \n",
       "month_lag_mean                      179986      1.47622       0.3308   \n",
       "month_lag_std                       153199     0.396778     0.227721   \n",
       "month_lag_var                       153199     0.209289     0.137089   \n",
       "purchase_date_amin                     NaN          NaN          NaN   \n",
       "purchase_date_amax                     NaN          NaN          NaN   \n",
       "purchase_date_ptp                   179986      29.0291      20.0683   \n",
       "card_id_size                        179986      6.77655      6.69458   \n",
       "purchased_on_weekend_sum            179986       2.0832      2.72596   \n",
       "purchased_on_weekend_mean           179986     0.292901     0.287023   \n",
       "month_diff_mean                     179986       11.668      2.41644   \n",
       "category_1_sum                      179986            0            0   \n",
       "category_1_mean                          0          NaN          NaN   \n",
       "inverse_avg_transactions_per_day    179986      5.01168      4.55844   \n",
       "days_since_last_transaction         179986      305.242      73.4597   \n",
       "\n",
       "                                            min          25%         50%  \\\n",
       "first_active_month                          NaN          NaN         NaN   \n",
       "card_id                                     NaN          NaN         NaN   \n",
       "feature_1                                     1            2           3   \n",
       "feature_2                                     1            1           2   \n",
       "feature_3                                     0            0           1   \n",
       "target                                 -33.2193     -0.88311  -0.0234369   \n",
       "first_active_monthYear                     2011         2016        2017   \n",
       "first_active_monthMonth                       1            5           8   \n",
       "first_active_monthWeek                        1           22          31   \n",
       "first_active_monthDay                         1            1           1   \n",
       "first_active_monthDayofweek                   0            2           3   \n",
       "first_active_monthDayofyear                   1          121         213   \n",
       "first_active_monthIs_month_end              NaN          NaN         NaN   \n",
       "first_active_monthIs_month_start            NaN          NaN         NaN   \n",
       "first_active_monthIs_quarter_end            NaN          NaN         NaN   \n",
       "first_active_monthIs_quarter_start          NaN          NaN         NaN   \n",
       "first_active_monthIs_year_end               NaN          NaN         NaN   \n",
       "first_active_monthIs_year_start             NaN          NaN         NaN   \n",
       "first_active_monthElapsed           1.32011e+09  1.47528e+09   1.491e+09   \n",
       "transactions_count                            1            2           5   \n",
       "city_id_nunique                               1            1           2   \n",
       "merchant_category_id_nunique                  1            2           4   \n",
       "merchant_id_nunique                           0            2           5   \n",
       "state_id_nunique                              1            1           1   \n",
       "purchase_Month_nunique                        1            1           2   \n",
       "subsector_id_nunique                          1            2           4   \n",
       "authorized_flag_mean                        NaN          NaN         NaN   \n",
       "authorized_flag_sum                           0            0           0   \n",
       "purchase_amount_sum                    -70.8135     -5.25733    -2.68625   \n",
       "purchase_amount_max                   -0.746893    -0.647883    -0.51302   \n",
       "purchase_amount_min                   -0.746893    -0.735638   -0.723466   \n",
       "purchase_amount_mean                  -0.746893    -0.692782   -0.641737   \n",
       "purchase_amount_std                           0     0.039286   0.0890018   \n",
       "purchase_amount_var                           0   0.00154339  0.00792133   \n",
       "installments_sum                             -6            0           1   \n",
       "installments_max                             -1            0           1   \n",
       "installments_min                             -1            0           0   \n",
       "installments_mean                            -1            0    0.333333   \n",
       "installments_std                              0            0           0   \n",
       "installments_var                              0            0           0   \n",
       "month_lag_sum                                 1            3           7   \n",
       "month_lag_max                                 1            2           2   \n",
       "month_lag_min                                 1            1           1   \n",
       "month_lag_mean                                1      1.23077         1.5   \n",
       "month_lag_std                                 0     0.351091         0.5   \n",
       "month_lag_var                                 0     0.123265        0.25   \n",
       "purchase_date_amin                          NaN          NaN         NaN   \n",
       "purchase_date_amax                          NaN          NaN         NaN   \n",
       "purchase_date_ptp                             0            8          32   \n",
       "card_id_size                                  1            2           5   \n",
       "purchased_on_weekend_sum                      0            0           1   \n",
       "purchased_on_weekend_mean                     0            0        0.25   \n",
       "month_diff_mean                              10         10.4     10.7143   \n",
       "category_1_sum                                0            0           0   \n",
       "category_1_mean                             NaN          NaN         NaN   \n",
       "inverse_avg_transactions_per_day              0      1.70588     4.14286   \n",
       "days_since_last_transaction                 255          261         276   \n",
       "\n",
       "                                            75%          max  counts uniques  \\\n",
       "first_active_month                          NaN          NaN  201917      75   \n",
       "card_id                                     NaN          NaN  201917  201917   \n",
       "feature_1                                     4            5  201917       5   \n",
       "feature_2                                     2            3  201917       3   \n",
       "feature_3                                     1            1  201917       2   \n",
       "target                                 0.765453      17.9651  201917  197110   \n",
       "first_active_monthYear                     2017         2018  201917       8   \n",
       "first_active_monthMonth                      10           12  201917      12   \n",
       "first_active_monthWeek                       44           53  201917      21   \n",
       "first_active_monthDay                         1            1  201917       1   \n",
       "first_active_monthDayofweek                   5            6  201917       7   \n",
       "first_active_monthDayofyear                 275          336  201917      22   \n",
       "first_active_monthIs_month_end              NaN          NaN  201917       1   \n",
       "first_active_monthIs_month_start            NaN          NaN  201917       1   \n",
       "first_active_monthIs_quarter_end            NaN          NaN  201917       1   \n",
       "first_active_monthIs_quarter_start          NaN          NaN  201917       2   \n",
       "first_active_monthIs_year_end               NaN          NaN  201917       1   \n",
       "first_active_monthIs_year_start             NaN          NaN  201917       2   \n",
       "first_active_monthElapsed           1.50422e+09  1.51744e+09  201917      75   \n",
       "transactions_count                            9          109  179986      87   \n",
       "city_id_nunique                               3           28  179986      26   \n",
       "merchant_category_id_nunique                  7           40  179986      39   \n",
       "merchant_id_nunique                           9          109  179986      87   \n",
       "state_id_nunique                              2           14  179986      13   \n",
       "purchase_Month_nunique                        2            2  179986       2   \n",
       "subsector_id_nunique                          6           24  179986      24   \n",
       "authorized_flag_mean                        NaN          NaN       0       0   \n",
       "authorized_flag_sum                           0            0  179986       1   \n",
       "purchase_amount_sum                    -1.21177      109.086  179986  149495   \n",
       "purchase_amount_max                   -0.181911      76.6829  179986   43853   \n",
       "purchase_amount_min                   -0.692076      44.1824  179986   15978   \n",
       "purchase_amount_mean                  -0.531653      44.1824  179986  146245   \n",
       "purchase_amount_std                    0.212109      47.3277  153199  145877   \n",
       "purchase_amount_var                     0.04499      2239.92  153199  146001   \n",
       "installments_sum                              7         1004  179986     107   \n",
       "installments_max                              2          999  179986      15   \n",
       "installments_min                              1           12  179986      14   \n",
       "installments_mean                             1      34.6207  179986    1104   \n",
       "installments_std                       0.755929      185.477  153199    7102   \n",
       "installments_var                       0.571429      34401.7  153199    8167   \n",
       "month_lag_sum                                13          185  179986     126   \n",
       "month_lag_max                                 2            2  179986       2   \n",
       "month_lag_min                                 1            2  179986       2   \n",
       "month_lag_mean                          1.71429            2  179986     690   \n",
       "month_lag_std                          0.534522     0.707107  153199    1473   \n",
       "month_lag_var                          0.285714          0.5  153199    1946   \n",
       "purchase_date_amin                          NaN          NaN  179986  172918   \n",
       "purchase_date_amax                          NaN          NaN  179986  173711   \n",
       "purchase_date_ptp                            47           61  179986      62   \n",
       "card_id_size                                  9          109  179986      87   \n",
       "purchased_on_weekend_sum                      3           45  179986      39   \n",
       "purchased_on_weekend_mean                   0.5            1  179986     579   \n",
       "month_diff_mean                            11.6           23  179986    2790   \n",
       "category_1_sum                                0            0  179986       1   \n",
       "category_1_mean                             NaN          NaN       0       0   \n",
       "inverse_avg_transactions_per_day          7.125           30  179986    1292   \n",
       "days_since_last_transaction                 310          678  179986     406   \n",
       "\n",
       "                                   missing missing_perc     types  \n",
       "first_active_month                       0           0%      date  \n",
       "card_id                                  0           0%    unique  \n",
       "feature_1                                0           0%   numeric  \n",
       "feature_2                                0           0%   numeric  \n",
       "feature_3                                0           0%      bool  \n",
       "target                                   0           0%   numeric  \n",
       "first_active_monthYear                   0           0%   numeric  \n",
       "first_active_monthMonth                  0           0%   numeric  \n",
       "first_active_monthWeek                   0           0%   numeric  \n",
       "first_active_monthDay                    0           0%  constant  \n",
       "first_active_monthDayofweek              0           0%   numeric  \n",
       "first_active_monthDayofyear              0           0%   numeric  \n",
       "first_active_monthIs_month_end           0           0%  constant  \n",
       "first_active_monthIs_month_start         0           0%  constant  \n",
       "first_active_monthIs_quarter_end         0           0%  constant  \n",
       "first_active_monthIs_quarter_start       0           0%      bool  \n",
       "first_active_monthIs_year_end            0           0%  constant  \n",
       "first_active_monthIs_year_start          0           0%      bool  \n",
       "first_active_monthElapsed                0           0%   numeric  \n",
       "transactions_count                   21931       10.86%   numeric  \n",
       "city_id_nunique                      21931       10.86%   numeric  \n",
       "merchant_category_id_nunique         21931       10.86%   numeric  \n",
       "merchant_id_nunique                  21931       10.86%   numeric  \n",
       "state_id_nunique                     21931       10.86%   numeric  \n",
       "purchase_Month_nunique               21931       10.86%      bool  \n",
       "subsector_id_nunique                 21931       10.86%   numeric  \n",
       "authorized_flag_mean                201917         100%   numeric  \n",
       "authorized_flag_sum                  21931       10.86%  constant  \n",
       "purchase_amount_sum                  21931       10.86%   numeric  \n",
       "purchase_amount_max                  21931       10.86%   numeric  \n",
       "purchase_amount_min                  21931       10.86%   numeric  \n",
       "purchase_amount_mean                 21931       10.86%   numeric  \n",
       "purchase_amount_std                  48718       24.13%   numeric  \n",
       "purchase_amount_var                  48718       24.13%   numeric  \n",
       "installments_sum                     21931       10.86%   numeric  \n",
       "installments_max                     21931       10.86%   numeric  \n",
       "installments_min                     21931       10.86%   numeric  \n",
       "installments_mean                    21931       10.86%   numeric  \n",
       "installments_std                     48718       24.13%   numeric  \n",
       "installments_var                     48718       24.13%   numeric  \n",
       "month_lag_sum                        21931       10.86%   numeric  \n",
       "month_lag_max                        21931       10.86%      bool  \n",
       "month_lag_min                        21931       10.86%      bool  \n",
       "month_lag_mean                       21931       10.86%   numeric  \n",
       "month_lag_std                        48718       24.13%   numeric  \n",
       "month_lag_var                        48718       24.13%   numeric  \n",
       "purchase_date_amin                   21931       10.86%      date  \n",
       "purchase_date_amax                   21931       10.86%      date  \n",
       "purchase_date_ptp                    21931       10.86%   numeric  \n",
       "card_id_size                         21931       10.86%   numeric  \n",
       "purchased_on_weekend_sum             21931       10.86%   numeric  \n",
       "purchased_on_weekend_mean            21931       10.86%   numeric  \n",
       "month_diff_mean                      21931       10.86%   numeric  \n",
       "category_1_sum                       21931       10.86%  constant  \n",
       "category_1_mean                     201917         100%   numeric  \n",
       "inverse_avg_transactions_per_day     21931       10.86%   numeric  \n",
       "days_since_last_transaction          21931       10.86%   numeric  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DataFrameSummary(train_df).summary().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def join_dfs(left, right, left_on, right_on=None, suffix='_old'):\n",
    "    if right_on is None: right_on = left_on\n",
    "    return left.merge(right, how='left', left_on=left_on, right_on=right_on, suffixes=(\"\", suffix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df2 = join_dfs(train_df, hist_trans_agg, left_on='card_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "      <th>counts</th>\n",
       "      <th>uniques</th>\n",
       "      <th>missing</th>\n",
       "      <th>missing_perc</th>\n",
       "      <th>types</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>first_active_month</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>unique</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1</th>\n",
       "      <td>201917</td>\n",
       "      <td>3.10531</td>\n",
       "      <td>1.18616</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>201917</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2</th>\n",
       "      <td>201917</td>\n",
       "      <td>1.74541</td>\n",
       "      <td>0.751362</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>201917</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_3</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.565569</td>\n",
       "      <td>0.495683</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>201917</td>\n",
       "      <td>-0.393636</td>\n",
       "      <td>3.8505</td>\n",
       "      <td>-33.2193</td>\n",
       "      <td>-0.88311</td>\n",
       "      <td>-0.0234369</td>\n",
       "      <td>0.765453</td>\n",
       "      <td>17.9651</td>\n",
       "      <td>201917</td>\n",
       "      <td>197110</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthYear</th>\n",
       "      <td>201917</td>\n",
       "      <td>2016.51</td>\n",
       "      <td>0.788199</td>\n",
       "      <td>2011</td>\n",
       "      <td>2016</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "      <td>2018</td>\n",
       "      <td>201917</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthMonth</th>\n",
       "      <td>201917</td>\n",
       "      <td>7.37874</td>\n",
       "      <td>3.34072</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>201917</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthWeek</th>\n",
       "      <td>201917</td>\n",
       "      <td>30.9446</td>\n",
       "      <td>13.8454</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>31</td>\n",
       "      <td>44</td>\n",
       "      <td>53</td>\n",
       "      <td>201917</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDay</th>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDayofweek</th>\n",
       "      <td>201917</td>\n",
       "      <td>3.17948</td>\n",
       "      <td>1.89371</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>201917</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDayofyear</th>\n",
       "      <td>201917</td>\n",
       "      <td>194.482</td>\n",
       "      <td>101.796</td>\n",
       "      <td>1</td>\n",
       "      <td>121</td>\n",
       "      <td>213</td>\n",
       "      <td>275</td>\n",
       "      <td>336</td>\n",
       "      <td>201917</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_month_end</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_month_start</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_quarter_end</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_quarter_start</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_year_end</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_year_start</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthElapsed</th>\n",
       "      <td>201917</td>\n",
       "      <td>1.48444e+09</td>\n",
       "      <td>2.53766e+07</td>\n",
       "      <td>1.32011e+09</td>\n",
       "      <td>1.47528e+09</td>\n",
       "      <td>1.491e+09</td>\n",
       "      <td>1.50422e+09</td>\n",
       "      <td>1.51744e+09</td>\n",
       "      <td>201917</td>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transactions_count</th>\n",
       "      <td>179986</td>\n",
       "      <td>6.77655</td>\n",
       "      <td>6.69458</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>109</td>\n",
       "      <td>179986</td>\n",
       "      <td>87</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city_id_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>2.30943</td>\n",
       "      <td>1.67011</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>28</td>\n",
       "      <td>179986</td>\n",
       "      <td>26</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_category_id_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>5.33923</td>\n",
       "      <td>4.25534</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>40</td>\n",
       "      <td>179986</td>\n",
       "      <td>39</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_id_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>6.68335</td>\n",
       "      <td>6.65009</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>109</td>\n",
       "      <td>179986</td>\n",
       "      <td>87</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_id_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>1.58205</td>\n",
       "      <td>0.873557</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>179986</td>\n",
       "      <td>13</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Month_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>1.65671</td>\n",
       "      <td>0.47481</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>179986</td>\n",
       "      <td>2</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsector_id_nunique</th>\n",
       "      <td>179986</td>\n",
       "      <td>4.45392</td>\n",
       "      <td>3.05844</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>179986</td>\n",
       "      <td>24</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_mean</th>\n",
       "      <td>179986</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>179986</td>\n",
       "      <td>1</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_sum</th>\n",
       "      <td>179986</td>\n",
       "      <td>6.77655</td>\n",
       "      <td>6.69458</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>109</td>\n",
       "      <td>179986</td>\n",
       "      <td>87</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_sum</th>\n",
       "      <td>179986</td>\n",
       "      <td>-3.74189</td>\n",
       "      <td>4.39531</td>\n",
       "      <td>-70.8135</td>\n",
       "      <td>-5.25733</td>\n",
       "      <td>-2.68625</td>\n",
       "      <td>-1.21177</td>\n",
       "      <td>109.086</td>\n",
       "      <td>179986</td>\n",
       "      <td>149495</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_max</th>\n",
       "      <td>179986</td>\n",
       "      <td>-0.131691</td>\n",
       "      <td>1.53213</td>\n",
       "      <td>-0.746893</td>\n",
       "      <td>-0.647883</td>\n",
       "      <td>-0.51302</td>\n",
       "      <td>-0.181911</td>\n",
       "      <td>76.6829</td>\n",
       "      <td>179986</td>\n",
       "      <td>43853</td>\n",
       "      <td>21931</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsector_id_nunique_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>11.4608</td>\n",
       "      <td>5.18197</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>34</td>\n",
       "      <td>201917</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.893353</td>\n",
       "      <td>0.114446</td>\n",
       "      <td>0.0304878</td>\n",
       "      <td>0.852941</td>\n",
       "      <td>0.926829</td>\n",
       "      <td>0.972516</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>11356</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_sum_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>81.559</td>\n",
       "      <td>99.2434</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>48</td>\n",
       "      <td>100</td>\n",
       "      <td>2537</td>\n",
       "      <td>201917</td>\n",
       "      <td>973</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_sum_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>14.4257</td>\n",
       "      <td>13449.2</td>\n",
       "      <td>-2088.01</td>\n",
       "      <td>-65.8473</td>\n",
       "      <td>-30.3869</td>\n",
       "      <td>-12.4694</td>\n",
       "      <td>6.0106e+06</td>\n",
       "      <td>201917</td>\n",
       "      <td>201614</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_max_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>66.6717</td>\n",
       "      <td>13448.5</td>\n",
       "      <td>-0.745315</td>\n",
       "      <td>-0.347203</td>\n",
       "      <td>0.123548</td>\n",
       "      <td>1.27716</td>\n",
       "      <td>6.0106e+06</td>\n",
       "      <td>201917</td>\n",
       "      <td>74899</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_min_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>-0.729993</td>\n",
       "      <td>0.0918995</td>\n",
       "      <td>-0.746908</td>\n",
       "      <td>-0.743902</td>\n",
       "      <td>-0.740296</td>\n",
       "      <td>-0.731881</td>\n",
       "      <td>28.5548</td>\n",
       "      <td>201917</td>\n",
       "      <td>7432</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>2.45943</td>\n",
       "      <td>1115.17</td>\n",
       "      <td>-0.745371</td>\n",
       "      <td>-0.679118</td>\n",
       "      <td>-0.621435</td>\n",
       "      <td>-0.501713</td>\n",
       "      <td>500883</td>\n",
       "      <td>201917</td>\n",
       "      <td>201640</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_std_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>12.6537</td>\n",
       "      <td>3865.24</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0766826</td>\n",
       "      <td>0.160484</td>\n",
       "      <td>0.361742</td>\n",
       "      <td>1.73511e+06</td>\n",
       "      <td>201917</td>\n",
       "      <td>201633</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_var_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>1.49402e+07</td>\n",
       "      <td>6.69991e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00588022</td>\n",
       "      <td>0.0257552</td>\n",
       "      <td>0.130857</td>\n",
       "      <td>3.01061e+12</td>\n",
       "      <td>201917</td>\n",
       "      <td>201640</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_sum_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>57.854</td>\n",
       "      <td>100.709</td>\n",
       "      <td>-9</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>77</td>\n",
       "      <td>2413</td>\n",
       "      <td>201917</td>\n",
       "      <td>1039</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_max_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>4.25316</td>\n",
       "      <td>22.9412</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>999</td>\n",
       "      <td>201917</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_min_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.0559933</td>\n",
       "      <td>0.723225</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>201917</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.863678</td>\n",
       "      <td>1.13581</td>\n",
       "      <td>-0.692308</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.34426</td>\n",
       "      <td>79.3077</td>\n",
       "      <td>201917</td>\n",
       "      <td>21136</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_std_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.79586</td>\n",
       "      <td>3.05789</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.288104</td>\n",
       "      <td>1.15283</td>\n",
       "      <td>276.342</td>\n",
       "      <td>201917</td>\n",
       "      <td>87084</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_var_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>9.98405</td>\n",
       "      <td>507.6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.083004</td>\n",
       "      <td>1.32901</td>\n",
       "      <td>76364.9</td>\n",
       "      <td>201917</td>\n",
       "      <td>93906</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_sum_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>-400.169</td>\n",
       "      <td>610.571</td>\n",
       "      <td>-14172</td>\n",
       "      <td>-471</td>\n",
       "      <td>-179</td>\n",
       "      <td>-66</td>\n",
       "      <td>-1</td>\n",
       "      <td>201917</td>\n",
       "      <td>4267</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_max_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>-0.166058</td>\n",
       "      <td>0.636972</td>\n",
       "      <td>-11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>201917</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_min_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>-7.87778</td>\n",
       "      <td>3.84331</td>\n",
       "      <td>-13</td>\n",
       "      <td>-12</td>\n",
       "      <td>-8</td>\n",
       "      <td>-4</td>\n",
       "      <td>-1</td>\n",
       "      <td>201917</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>-3.84099</td>\n",
       "      <td>2.12349</td>\n",
       "      <td>-12.4524</td>\n",
       "      <td>-5.52062</td>\n",
       "      <td>-3.57143</td>\n",
       "      <td>-2.03497</td>\n",
       "      <td>-0.0149254</td>\n",
       "      <td>201917</td>\n",
       "      <td>54905</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_std_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>2.40891</td>\n",
       "      <td>1.17382</td>\n",
       "      <td>0.122169</td>\n",
       "      <td>1.37663</td>\n",
       "      <td>2.2734</td>\n",
       "      <td>3.44656</td>\n",
       "      <td>7</td>\n",
       "      <td>201917</td>\n",
       "      <td>160944</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_var_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>7.1807</td>\n",
       "      <td>6.13092</td>\n",
       "      <td>0.0149254</td>\n",
       "      <td>1.89511</td>\n",
       "      <td>5.16835</td>\n",
       "      <td>11.8788</td>\n",
       "      <td>49</td>\n",
       "      <td>201917</td>\n",
       "      <td>167801</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amin_old</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>193984</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amax_old</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>186997</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_ptp_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>243.678</td>\n",
       "      <td>118.292</td>\n",
       "      <td>1</td>\n",
       "      <td>138</td>\n",
       "      <td>233</td>\n",
       "      <td>361</td>\n",
       "      <td>423</td>\n",
       "      <td>201917</td>\n",
       "      <td>422</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id_size_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>89.2942</td>\n",
       "      <td>104.606</td>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>55</td>\n",
       "      <td>111</td>\n",
       "      <td>2912</td>\n",
       "      <td>201917</td>\n",
       "      <td>1023</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_sum_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>24.9963</td>\n",
       "      <td>31.7337</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>31</td>\n",
       "      <td>659</td>\n",
       "      <td>201917</td>\n",
       "      <td>379</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.276675</td>\n",
       "      <td>0.138688</td>\n",
       "      <td>0</td>\n",
       "      <td>0.186047</td>\n",
       "      <td>0.269663</td>\n",
       "      <td>0.356522</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>15633</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>11.6532</td>\n",
       "      <td>2.39774</td>\n",
       "      <td>10</td>\n",
       "      <td>10.4444</td>\n",
       "      <td>10.5714</td>\n",
       "      <td>11.55</td>\n",
       "      <td>23</td>\n",
       "      <td>201917</td>\n",
       "      <td>33406</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inverse_avg_transactions_per_day_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>6.07849</td>\n",
       "      <td>6.93816</td>\n",
       "      <td>0.0869565</td>\n",
       "      <td>2</td>\n",
       "      <td>3.76147</td>\n",
       "      <td>7.36</td>\n",
       "      <td>135.667</td>\n",
       "      <td>201917</td>\n",
       "      <td>45200</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_since_last_transaction_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>360.615</td>\n",
       "      <td>74.7399</td>\n",
       "      <td>314</td>\n",
       "      <td>316</td>\n",
       "      <td>325</td>\n",
       "      <td>373</td>\n",
       "      <td>707</td>\n",
       "      <td>201917</td>\n",
       "      <td>392</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>91 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       count         mean          std  \\\n",
       "first_active_month                       NaN          NaN          NaN   \n",
       "card_id                                  NaN          NaN          NaN   \n",
       "feature_1                             201917      3.10531      1.18616   \n",
       "feature_2                             201917      1.74541     0.751362   \n",
       "feature_3                             201917     0.565569     0.495683   \n",
       "target                                201917    -0.393636       3.8505   \n",
       "first_active_monthYear                201917      2016.51     0.788199   \n",
       "first_active_monthMonth               201917      7.37874      3.34072   \n",
       "first_active_monthWeek                201917      30.9446      13.8454   \n",
       "first_active_monthDay                 201917            1            0   \n",
       "first_active_monthDayofweek           201917      3.17948      1.89371   \n",
       "first_active_monthDayofyear           201917      194.482      101.796   \n",
       "first_active_monthIs_month_end           NaN          NaN          NaN   \n",
       "first_active_monthIs_month_start         NaN          NaN          NaN   \n",
       "first_active_monthIs_quarter_end         NaN          NaN          NaN   \n",
       "first_active_monthIs_quarter_start       NaN          NaN          NaN   \n",
       "first_active_monthIs_year_end            NaN          NaN          NaN   \n",
       "first_active_monthIs_year_start          NaN          NaN          NaN   \n",
       "first_active_monthElapsed             201917  1.48444e+09  2.53766e+07   \n",
       "transactions_count                    179986      6.77655      6.69458   \n",
       "city_id_nunique                       179986      2.30943      1.67011   \n",
       "merchant_category_id_nunique          179986      5.33923      4.25534   \n",
       "merchant_id_nunique                   179986      6.68335      6.65009   \n",
       "state_id_nunique                      179986      1.58205     0.873557   \n",
       "purchase_Month_nunique                179986      1.65671      0.47481   \n",
       "subsector_id_nunique                  179986      4.45392      3.05844   \n",
       "authorized_flag_mean                  179986            1            0   \n",
       "authorized_flag_sum                   179986      6.77655      6.69458   \n",
       "purchase_amount_sum                   179986     -3.74189      4.39531   \n",
       "purchase_amount_max                   179986    -0.131691      1.53213   \n",
       "...                                      ...          ...          ...   \n",
       "subsector_id_nunique_old              201917      11.4608      5.18197   \n",
       "authorized_flag_mean_old              201917     0.893353     0.114446   \n",
       "authorized_flag_sum_old               201917       81.559      99.2434   \n",
       "purchase_amount_sum_old               201917      14.4257      13449.2   \n",
       "purchase_amount_max_old               201917      66.6717      13448.5   \n",
       "purchase_amount_min_old               201917    -0.729993    0.0918995   \n",
       "purchase_amount_mean_old              201917      2.45943      1115.17   \n",
       "purchase_amount_std_old               201917      12.6537      3865.24   \n",
       "purchase_amount_var_old               201917  1.49402e+07  6.69991e+09   \n",
       "installments_sum_old                  201917       57.854      100.709   \n",
       "installments_max_old                  201917      4.25316      22.9412   \n",
       "installments_min_old                  201917    0.0559933     0.723225   \n",
       "installments_mean_old                 201917     0.863678      1.13581   \n",
       "installments_std_old                  201917      0.79586      3.05789   \n",
       "installments_var_old                  201917      9.98405        507.6   \n",
       "month_lag_sum_old                     201917     -400.169      610.571   \n",
       "month_lag_max_old                     201917    -0.166058     0.636972   \n",
       "month_lag_min_old                     201917     -7.87778      3.84331   \n",
       "month_lag_mean_old                    201917     -3.84099      2.12349   \n",
       "month_lag_std_old                     201917      2.40891      1.17382   \n",
       "month_lag_var_old                     201917       7.1807      6.13092   \n",
       "purchase_date_amin_old                   NaN          NaN          NaN   \n",
       "purchase_date_amax_old                   NaN          NaN          NaN   \n",
       "purchase_date_ptp_old                 201917      243.678      118.292   \n",
       "card_id_size_old                      201917      89.2942      104.606   \n",
       "purchased_on_weekend_sum_old          201917      24.9963      31.7337   \n",
       "purchased_on_weekend_mean_old         201917     0.276675     0.138688   \n",
       "month_diff_mean_old                   201917      11.6532      2.39774   \n",
       "inverse_avg_transactions_per_day_old  201917      6.07849      6.93816   \n",
       "days_since_last_transaction_old       201917      360.615      74.7399   \n",
       "\n",
       "                                              min          25%        50%  \\\n",
       "first_active_month                            NaN          NaN        NaN   \n",
       "card_id                                       NaN          NaN        NaN   \n",
       "feature_1                                       1            2          3   \n",
       "feature_2                                       1            1          2   \n",
       "feature_3                                       0            0          1   \n",
       "target                                   -33.2193     -0.88311 -0.0234369   \n",
       "first_active_monthYear                       2011         2016       2017   \n",
       "first_active_monthMonth                         1            5          8   \n",
       "first_active_monthWeek                          1           22         31   \n",
       "first_active_monthDay                           1            1          1   \n",
       "first_active_monthDayofweek                     0            2          3   \n",
       "first_active_monthDayofyear                     1          121        213   \n",
       "first_active_monthIs_month_end                NaN          NaN        NaN   \n",
       "first_active_monthIs_month_start              NaN          NaN        NaN   \n",
       "first_active_monthIs_quarter_end              NaN          NaN        NaN   \n",
       "first_active_monthIs_quarter_start            NaN          NaN        NaN   \n",
       "first_active_monthIs_year_end                 NaN          NaN        NaN   \n",
       "first_active_monthIs_year_start               NaN          NaN        NaN   \n",
       "first_active_monthElapsed             1.32011e+09  1.47528e+09  1.491e+09   \n",
       "transactions_count                              1            2          5   \n",
       "city_id_nunique                                 1            1          2   \n",
       "merchant_category_id_nunique                    1            2          4   \n",
       "merchant_id_nunique                             0            2          5   \n",
       "state_id_nunique                                1            1          1   \n",
       "purchase_Month_nunique                          1            1          2   \n",
       "subsector_id_nunique                            1            2          4   \n",
       "authorized_flag_mean                            1            1          1   \n",
       "authorized_flag_sum                             1            2          5   \n",
       "purchase_amount_sum                      -70.8135     -5.25733   -2.68625   \n",
       "purchase_amount_max                     -0.746893    -0.647883   -0.51302   \n",
       "...                                           ...          ...        ...   \n",
       "subsector_id_nunique_old                        1            8         11   \n",
       "authorized_flag_mean_old                0.0304878     0.852941   0.926829   \n",
       "authorized_flag_sum_old                         2           23         48   \n",
       "purchase_amount_sum_old                  -2088.01     -65.8473   -30.3869   \n",
       "purchase_amount_max_old                 -0.745315    -0.347203   0.123548   \n",
       "purchase_amount_min_old                 -0.746908    -0.743902  -0.740296   \n",
       "purchase_amount_mean_old                -0.745371    -0.679118  -0.621435   \n",
       "purchase_amount_std_old                         0    0.0766826   0.160484   \n",
       "purchase_amount_var_old                         0   0.00588022  0.0257552   \n",
       "installments_sum_old                           -9            0         15   \n",
       "installments_max_old                            0            0          2   \n",
       "installments_min_old                           -1            0          0   \n",
       "installments_mean_old                   -0.692308            0          1   \n",
       "installments_std_old                            0            0   0.288104   \n",
       "installments_var_old                            0            0   0.083004   \n",
       "month_lag_sum_old                          -14172         -471       -179   \n",
       "month_lag_max_old                             -11            0          0   \n",
       "month_lag_min_old                             -13          -12         -8   \n",
       "month_lag_mean_old                       -12.4524     -5.52062   -3.57143   \n",
       "month_lag_std_old                        0.122169      1.37663     2.2734   \n",
       "month_lag_var_old                       0.0149254      1.89511    5.16835   \n",
       "purchase_date_amin_old                        NaN          NaN        NaN   \n",
       "purchase_date_amax_old                        NaN          NaN        NaN   \n",
       "purchase_date_ptp_old                           1          138        233   \n",
       "card_id_size_old                                2           26         55   \n",
       "purchased_on_weekend_sum_old                    0            6         14   \n",
       "purchased_on_weekend_mean_old                   0     0.186047   0.269663   \n",
       "month_diff_mean_old                            10      10.4444    10.5714   \n",
       "inverse_avg_transactions_per_day_old    0.0869565            2    3.76147   \n",
       "days_since_last_transaction_old               314          316        325   \n",
       "\n",
       "                                              75%          max  counts  \\\n",
       "first_active_month                            NaN          NaN  201917   \n",
       "card_id                                       NaN          NaN  201917   \n",
       "feature_1                                       4            5  201917   \n",
       "feature_2                                       2            3  201917   \n",
       "feature_3                                       1            1  201917   \n",
       "target                                   0.765453      17.9651  201917   \n",
       "first_active_monthYear                       2017         2018  201917   \n",
       "first_active_monthMonth                        10           12  201917   \n",
       "first_active_monthWeek                         44           53  201917   \n",
       "first_active_monthDay                           1            1  201917   \n",
       "first_active_monthDayofweek                     5            6  201917   \n",
       "first_active_monthDayofyear                   275          336  201917   \n",
       "first_active_monthIs_month_end                NaN          NaN  201917   \n",
       "first_active_monthIs_month_start              NaN          NaN  201917   \n",
       "first_active_monthIs_quarter_end              NaN          NaN  201917   \n",
       "first_active_monthIs_quarter_start            NaN          NaN  201917   \n",
       "first_active_monthIs_year_end                 NaN          NaN  201917   \n",
       "first_active_monthIs_year_start               NaN          NaN  201917   \n",
       "first_active_monthElapsed             1.50422e+09  1.51744e+09  201917   \n",
       "transactions_count                              9          109  179986   \n",
       "city_id_nunique                                 3           28  179986   \n",
       "merchant_category_id_nunique                    7           40  179986   \n",
       "merchant_id_nunique                             9          109  179986   \n",
       "state_id_nunique                                2           14  179986   \n",
       "purchase_Month_nunique                          2            2  179986   \n",
       "subsector_id_nunique                            6           24  179986   \n",
       "authorized_flag_mean                            1            1  179986   \n",
       "authorized_flag_sum                             9          109  179986   \n",
       "purchase_amount_sum                      -1.21177      109.086  179986   \n",
       "purchase_amount_max                     -0.181911      76.6829  179986   \n",
       "...                                           ...          ...     ...   \n",
       "subsector_id_nunique_old                       15           34  201917   \n",
       "authorized_flag_mean_old                 0.972516            1  201917   \n",
       "authorized_flag_sum_old                       100         2537  201917   \n",
       "purchase_amount_sum_old                  -12.4694   6.0106e+06  201917   \n",
       "purchase_amount_max_old                   1.27716   6.0106e+06  201917   \n",
       "purchase_amount_min_old                 -0.731881      28.5548  201917   \n",
       "purchase_amount_mean_old                -0.501713       500883  201917   \n",
       "purchase_amount_std_old                  0.361742  1.73511e+06  201917   \n",
       "purchase_amount_var_old                  0.130857  3.01061e+12  201917   \n",
       "installments_sum_old                           77         2413  201917   \n",
       "installments_max_old                            6          999  201917   \n",
       "installments_min_old                            1           10  201917   \n",
       "installments_mean_old                     1.34426      79.3077  201917   \n",
       "installments_std_old                      1.15283      276.342  201917   \n",
       "installments_var_old                      1.32901      76364.9  201917   \n",
       "month_lag_sum_old                             -66           -1  201917   \n",
       "month_lag_max_old                               0            0  201917   \n",
       "month_lag_min_old                              -4           -1  201917   \n",
       "month_lag_mean_old                       -2.03497   -0.0149254  201917   \n",
       "month_lag_std_old                         3.44656            7  201917   \n",
       "month_lag_var_old                         11.8788           49  201917   \n",
       "purchase_date_amin_old                        NaN          NaN  201917   \n",
       "purchase_date_amax_old                        NaN          NaN  201917   \n",
       "purchase_date_ptp_old                         361          423  201917   \n",
       "card_id_size_old                              111         2912  201917   \n",
       "purchased_on_weekend_sum_old                   31          659  201917   \n",
       "purchased_on_weekend_mean_old            0.356522            1  201917   \n",
       "month_diff_mean_old                         11.55           23  201917   \n",
       "inverse_avg_transactions_per_day_old         7.36      135.667  201917   \n",
       "days_since_last_transaction_old               373          707  201917   \n",
       "\n",
       "                                     uniques missing missing_perc     types  \n",
       "first_active_month                        75       0           0%      date  \n",
       "card_id                               201917       0           0%    unique  \n",
       "feature_1                                  5       0           0%   numeric  \n",
       "feature_2                                  3       0           0%   numeric  \n",
       "feature_3                                  2       0           0%      bool  \n",
       "target                                197110       0           0%   numeric  \n",
       "first_active_monthYear                     8       0           0%   numeric  \n",
       "first_active_monthMonth                   12       0           0%   numeric  \n",
       "first_active_monthWeek                    21       0           0%   numeric  \n",
       "first_active_monthDay                      1       0           0%  constant  \n",
       "first_active_monthDayofweek                7       0           0%   numeric  \n",
       "first_active_monthDayofyear               22       0           0%   numeric  \n",
       "first_active_monthIs_month_end             1       0           0%  constant  \n",
       "first_active_monthIs_month_start           1       0           0%  constant  \n",
       "first_active_monthIs_quarter_end           1       0           0%  constant  \n",
       "first_active_monthIs_quarter_start         2       0           0%      bool  \n",
       "first_active_monthIs_year_end              1       0           0%  constant  \n",
       "first_active_monthIs_year_start            2       0           0%      bool  \n",
       "first_active_monthElapsed                 75       0           0%   numeric  \n",
       "transactions_count                        87   21931       10.86%   numeric  \n",
       "city_id_nunique                           26   21931       10.86%   numeric  \n",
       "merchant_category_id_nunique              39   21931       10.86%   numeric  \n",
       "merchant_id_nunique                       87   21931       10.86%   numeric  \n",
       "state_id_nunique                          13   21931       10.86%   numeric  \n",
       "purchase_Month_nunique                     2   21931       10.86%      bool  \n",
       "subsector_id_nunique                      24   21931       10.86%   numeric  \n",
       "authorized_flag_mean                       1   21931       10.86%  constant  \n",
       "authorized_flag_sum                       87   21931       10.86%   numeric  \n",
       "purchase_amount_sum                   149495   21931       10.86%   numeric  \n",
       "purchase_amount_max                    43853   21931       10.86%   numeric  \n",
       "...                                      ...     ...          ...       ...  \n",
       "subsector_id_nunique_old                  34       0           0%   numeric  \n",
       "authorized_flag_mean_old               11356       0           0%   numeric  \n",
       "authorized_flag_sum_old                  973       0           0%   numeric  \n",
       "purchase_amount_sum_old               201614       0           0%   numeric  \n",
       "purchase_amount_max_old                74899       0           0%   numeric  \n",
       "purchase_amount_min_old                 7432       0           0%   numeric  \n",
       "purchase_amount_mean_old              201640       0           0%   numeric  \n",
       "purchase_amount_std_old               201633       0           0%   numeric  \n",
       "purchase_amount_var_old               201640       0           0%   numeric  \n",
       "installments_sum_old                    1039       0           0%   numeric  \n",
       "installments_max_old                      14       0           0%   numeric  \n",
       "installments_min_old                      10       0           0%   numeric  \n",
       "installments_mean_old                  21136       0           0%   numeric  \n",
       "installments_std_old                   87084       0           0%   numeric  \n",
       "installments_var_old                   93906       0           0%   numeric  \n",
       "month_lag_sum_old                       4267       0           0%   numeric  \n",
       "month_lag_max_old                         12       0           0%   numeric  \n",
       "month_lag_min_old                         13       0           0%   numeric  \n",
       "month_lag_mean_old                     54905       0           0%   numeric  \n",
       "month_lag_std_old                     160944       0           0%   numeric  \n",
       "month_lag_var_old                     167801       0           0%   numeric  \n",
       "purchase_date_amin_old                193984       0           0%      date  \n",
       "purchase_date_amax_old                186997       0           0%      date  \n",
       "purchase_date_ptp_old                    422       0           0%   numeric  \n",
       "card_id_size_old                        1023       0           0%   numeric  \n",
       "purchased_on_weekend_sum_old             379       0           0%   numeric  \n",
       "purchased_on_weekend_mean_old          15633       0           0%   numeric  \n",
       "month_diff_mean_old                    33406       0           0%   numeric  \n",
       "inverse_avg_transactions_per_day_old   45200       0           0%   numeric  \n",
       "days_since_last_transaction_old          392       0           0%   numeric  \n",
       "\n",
       "[91 rows x 13 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DataFrameSummary(train_df2).summary().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_all(df):\n",
    "    with pd.option_context(\"display.max_rows\", 100, \"display.max_columns\", 100): \n",
    "        display(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>first_active_month</th>\n",
       "      <td>2017-06-01 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <td>C_ID_92a2005557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_3</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>-0.820283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthYear</th>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthMonth</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthWeek</th>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDay</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDayofweek</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDayofyear</th>\n",
       "      <td>152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_month_end</th>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_month_start</th>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_quarter_end</th>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_quarter_start</th>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_year_end</th>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_year_start</th>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthElapsed</th>\n",
       "      <td>1496275200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transactions_count</th>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city_id_nunique</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_category_id_nunique</th>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_id_nunique</th>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_id_nunique</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Month_nunique</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsector_id_nunique</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_mean</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_sum</th>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_sum</th>\n",
       "      <td>-13.2442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_max</th>\n",
       "      <td>-0.296112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_min</th>\n",
       "      <td>-0.724368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_mean</th>\n",
       "      <td>-0.575835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_std</th>\n",
       "      <td>0.135812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_var</th>\n",
       "      <td>0.0184448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_sum</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_max</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_min</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_mean</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_std</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_var</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_sum</th>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_max</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_min</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_mean</th>\n",
       "      <td>1.47826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_std</th>\n",
       "      <td>0.510754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_var</th>\n",
       "      <td>0.26087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amin</th>\n",
       "      <td>2018-03-05 14:04:36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amax</th>\n",
       "      <td>2018-04-29 11:23:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_ptp</th>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id_size</th>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_sum</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_mean</th>\n",
       "      <td>0.26087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff_mean</th>\n",
       "      <td>10.5217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inverse_avg_transactions_per_day</th>\n",
       "      <td>2.34783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_since_last_transaction</th>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transactions_count_old</th>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city_id_nunique_old</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_category_id_nunique_old</th>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_id_nunique_old</th>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_id_nunique_old</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Month_nunique_old</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsector_id_nunique_old</th>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_mean_old</th>\n",
       "      <td>0.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_sum_old</th>\n",
       "      <td>247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_sum_old</th>\n",
       "      <td>-165.969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_max_old</th>\n",
       "      <td>2.25839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_min_old</th>\n",
       "      <td>-0.739395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_mean_old</th>\n",
       "      <td>-0.638341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_std_old</th>\n",
       "      <td>0.212139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_var_old</th>\n",
       "      <td>0.0450031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_sum_old</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_max_old</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_min_old</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_mean_old</th>\n",
       "      <td>0.0153846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_std_old</th>\n",
       "      <td>0.123314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_var_old</th>\n",
       "      <td>0.0152064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_sum_old</th>\n",
       "      <td>-1017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_max_old</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_min_old</th>\n",
       "      <td>-8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_mean_old</th>\n",
       "      <td>-3.91154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_std_old</th>\n",
       "      <td>2.39769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_var_old</th>\n",
       "      <td>5.7489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amin_old</th>\n",
       "      <td>2017-06-27 14:18:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amax_old</th>\n",
       "      <td>2018-02-25 09:31:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_ptp_old</th>\n",
       "      <td>242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id_size_old</th>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_sum_old</th>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_mean_old</th>\n",
       "      <td>0.346154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff_mean_old</th>\n",
       "      <td>10.5423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inverse_avg_transactions_per_day_old</th>\n",
       "      <td>0.930769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_since_last_transaction_old</th>\n",
       "      <td>318</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                        0\n",
       "first_active_month                    2017-06-01 00:00:00\n",
       "card_id                                   C_ID_92a2005557\n",
       "feature_1                                               5\n",
       "feature_2                                               2\n",
       "feature_3                                               1\n",
       "target                                          -0.820283\n",
       "first_active_monthYear                               2017\n",
       "first_active_monthMonth                                 6\n",
       "first_active_monthWeek                                 22\n",
       "first_active_monthDay                                   1\n",
       "first_active_monthDayofweek                             3\n",
       "first_active_monthDayofyear                           152\n",
       "first_active_monthIs_month_end                      False\n",
       "first_active_monthIs_month_start                     True\n",
       "first_active_monthIs_quarter_end                    False\n",
       "first_active_monthIs_quarter_start                  False\n",
       "first_active_monthIs_year_end                       False\n",
       "first_active_monthIs_year_start                     False\n",
       "first_active_monthElapsed                      1496275200\n",
       "transactions_count                                     23\n",
       "city_id_nunique                                         3\n",
       "merchant_category_id_nunique                           14\n",
       "merchant_id_nunique                                    23\n",
       "state_id_nunique                                        1\n",
       "purchase_Month_nunique                                  2\n",
       "subsector_id_nunique                                   10\n",
       "authorized_flag_mean                                    1\n",
       "authorized_flag_sum                                    23\n",
       "purchase_amount_sum                              -13.2442\n",
       "purchase_amount_max                             -0.296112\n",
       "purchase_amount_min                             -0.724368\n",
       "purchase_amount_mean                            -0.575835\n",
       "purchase_amount_std                              0.135812\n",
       "purchase_amount_var                             0.0184448\n",
       "installments_sum                                        0\n",
       "installments_max                                        0\n",
       "installments_min                                        0\n",
       "installments_mean                                       0\n",
       "installments_std                                        0\n",
       "installments_var                                        0\n",
       "month_lag_sum                                          34\n",
       "month_lag_max                                           2\n",
       "month_lag_min                                           1\n",
       "month_lag_mean                                    1.47826\n",
       "month_lag_std                                    0.510754\n",
       "month_lag_var                                     0.26087\n",
       "purchase_date_amin                    2018-03-05 14:04:36\n",
       "purchase_date_amax                    2018-04-29 11:23:05\n",
       "purchase_date_ptp                                      54\n",
       "card_id_size                                           23\n",
       "purchased_on_weekend_sum                                6\n",
       "purchased_on_weekend_mean                         0.26087\n",
       "month_diff_mean                                   10.5217\n",
       "inverse_avg_transactions_per_day                  2.34783\n",
       "days_since_last_transaction                           255\n",
       "transactions_count_old                                260\n",
       "city_id_nunique_old                                     7\n",
       "merchant_category_id_nunique_old                       41\n",
       "merchant_id_nunique_old                                94\n",
       "state_id_nunique_old                                    3\n",
       "purchase_Month_nunique_old                              9\n",
       "subsector_id_nunique_old                               21\n",
       "authorized_flag_mean_old                             0.95\n",
       "authorized_flag_sum_old                               247\n",
       "purchase_amount_sum_old                          -165.969\n",
       "purchase_amount_max_old                           2.25839\n",
       "purchase_amount_min_old                         -0.739395\n",
       "purchase_amount_mean_old                        -0.638341\n",
       "purchase_amount_std_old                          0.212139\n",
       "purchase_amount_var_old                         0.0450031\n",
       "installments_sum_old                                    4\n",
       "installments_max_old                                    1\n",
       "installments_min_old                                    0\n",
       "installments_mean_old                           0.0153846\n",
       "installments_std_old                             0.123314\n",
       "installments_var_old                            0.0152064\n",
       "month_lag_sum_old                                   -1017\n",
       "month_lag_max_old                                       0\n",
       "month_lag_min_old                                      -8\n",
       "month_lag_mean_old                               -3.91154\n",
       "month_lag_std_old                                 2.39769\n",
       "month_lag_var_old                                  5.7489\n",
       "purchase_date_amin_old                2017-06-27 14:18:08\n",
       "purchase_date_amax_old                2018-02-25 09:31:15\n",
       "purchase_date_ptp_old                                 242\n",
       "card_id_size_old                                      260\n",
       "purchased_on_weekend_sum_old                           90\n",
       "purchased_on_weekend_mean_old                    0.346154\n",
       "month_diff_mean_old                               10.5423\n",
       "inverse_avg_transactions_per_day_old             0.930769\n",
       "days_since_last_transaction_old                       318"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_all(train_df2.head(1).T)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function to add extra cols on train & test combined agg dfs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Add total installments, purchase_amount, transactions count for the card on all transactions so far\n",
    "2. Age of the card \n",
    "3. difference b/w the card issue and first transaction\n",
    "4. Difference b/w card issue and last transaction - card active for how many days\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.merge(test, new_hist_agg, how='left', on='card_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df2 = join_dfs(test_df, hist_trans_agg, left_on='card_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(123623, 94)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(201917, 95)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "      <th>counts</th>\n",
       "      <th>uniques</th>\n",
       "      <th>missing</th>\n",
       "      <th>missing_perc</th>\n",
       "      <th>types</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>first_active_month</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>123622</td>\n",
       "      <td>75</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00%</td>\n",
       "      <td>date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>123623</td>\n",
       "      <td>123623</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>unique</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1</th>\n",
       "      <td>123623</td>\n",
       "      <td>3.10926</td>\n",
       "      <td>1.18911</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>123623</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2</th>\n",
       "      <td>123623</td>\n",
       "      <td>1.7418</td>\n",
       "      <td>0.749195</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>123623</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_3</th>\n",
       "      <td>123623</td>\n",
       "      <td>0.564377</td>\n",
       "      <td>0.49584</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>123623</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthYear</th>\n",
       "      <td>123622</td>\n",
       "      <td>2016.51</td>\n",
       "      <td>0.785047</td>\n",
       "      <td>2011</td>\n",
       "      <td>2016</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "      <td>2018</td>\n",
       "      <td>123622</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthMonth</th>\n",
       "      <td>123622</td>\n",
       "      <td>7.38177</td>\n",
       "      <td>3.3375</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>123622</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthWeek</th>\n",
       "      <td>123622</td>\n",
       "      <td>30.9737</td>\n",
       "      <td>13.8209</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>31</td>\n",
       "      <td>44</td>\n",
       "      <td>53</td>\n",
       "      <td>123622</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDay</th>\n",
       "      <td>123622</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>123622</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDayofweek</th>\n",
       "      <td>123622</td>\n",
       "      <td>3.18874</td>\n",
       "      <td>1.8966</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>123622</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDayofyear</th>\n",
       "      <td>123622</td>\n",
       "      <td>194.574</td>\n",
       "      <td>101.7</td>\n",
       "      <td>1</td>\n",
       "      <td>121</td>\n",
       "      <td>213</td>\n",
       "      <td>274</td>\n",
       "      <td>336</td>\n",
       "      <td>123622</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_month_end</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>123623</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_month_start</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>123623</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_quarter_end</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>123623</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_quarter_start</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>123623</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_year_end</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>123623</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_year_start</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>123623</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthElapsed</th>\n",
       "      <td>123623</td>\n",
       "      <td>1.48443e+09</td>\n",
       "      <td>3.95779e+07</td>\n",
       "      <td>-9.22337e+09</td>\n",
       "      <td>1.47528e+09</td>\n",
       "      <td>1.491e+09</td>\n",
       "      <td>1.50422e+09</td>\n",
       "      <td>1.51476e+09</td>\n",
       "      <td>123623</td>\n",
       "      <td>76</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transactions_count</th>\n",
       "      <td>110015</td>\n",
       "      <td>6.75677</td>\n",
       "      <td>6.65022</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>104</td>\n",
       "      <td>110015</td>\n",
       "      <td>80</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city_id_nunique</th>\n",
       "      <td>110015</td>\n",
       "      <td>2.30822</td>\n",
       "      <td>1.65098</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>32</td>\n",
       "      <td>110015</td>\n",
       "      <td>24</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_category_id_nunique</th>\n",
       "      <td>110015</td>\n",
       "      <td>5.33326</td>\n",
       "      <td>4.25539</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>46</td>\n",
       "      <td>110015</td>\n",
       "      <td>39</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_id_nunique</th>\n",
       "      <td>110015</td>\n",
       "      <td>6.66368</td>\n",
       "      <td>6.60652</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>103</td>\n",
       "      <td>110015</td>\n",
       "      <td>79</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_id_nunique</th>\n",
       "      <td>110015</td>\n",
       "      <td>1.58358</td>\n",
       "      <td>0.873022</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>110015</td>\n",
       "      <td>14</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Month_nunique</th>\n",
       "      <td>110015</td>\n",
       "      <td>1.65595</td>\n",
       "      <td>0.475061</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>110015</td>\n",
       "      <td>2</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsector_id_nunique</th>\n",
       "      <td>110015</td>\n",
       "      <td>4.4494</td>\n",
       "      <td>3.05787</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>23</td>\n",
       "      <td>110015</td>\n",
       "      <td>23</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_mean</th>\n",
       "      <td>110015</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>110015</td>\n",
       "      <td>1</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>constant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_sum</th>\n",
       "      <td>110015</td>\n",
       "      <td>6.75677</td>\n",
       "      <td>6.65022</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>104</td>\n",
       "      <td>110015</td>\n",
       "      <td>80</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_sum</th>\n",
       "      <td>110015</td>\n",
       "      <td>-3.70933</td>\n",
       "      <td>4.50389</td>\n",
       "      <td>-60.7804</td>\n",
       "      <td>-5.21013</td>\n",
       "      <td>-2.66773</td>\n",
       "      <td>-1.2066</td>\n",
       "      <td>263.157</td>\n",
       "      <td>110015</td>\n",
       "      <td>94244</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_max</th>\n",
       "      <td>110015</td>\n",
       "      <td>-0.122972</td>\n",
       "      <td>1.76015</td>\n",
       "      <td>-0.746893</td>\n",
       "      <td>-0.647883</td>\n",
       "      <td>-0.511593</td>\n",
       "      <td>-0.17593</td>\n",
       "      <td>263.157</td>\n",
       "      <td>110015</td>\n",
       "      <td>32279</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_min</th>\n",
       "      <td>110015</td>\n",
       "      <td>-0.656742</td>\n",
       "      <td>0.912294</td>\n",
       "      <td>-0.746893</td>\n",
       "      <td>-0.735638</td>\n",
       "      <td>-0.723617</td>\n",
       "      <td>-0.69161</td>\n",
       "      <td>263.157</td>\n",
       "      <td>110015</td>\n",
       "      <td>12517</td>\n",
       "      <td>13608</td>\n",
       "      <td>11.01%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsector_id_nunique_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>11.4836</td>\n",
       "      <td>5.19046</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>34</td>\n",
       "      <td>123623</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_mean_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>0.893012</td>\n",
       "      <td>0.114518</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.851852</td>\n",
       "      <td>0.926531</td>\n",
       "      <td>0.972603</td>\n",
       "      <td>1</td>\n",
       "      <td>123623</td>\n",
       "      <td>9131</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_sum_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>81.9209</td>\n",
       "      <td>100.695</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>48</td>\n",
       "      <td>100</td>\n",
       "      <td>4122</td>\n",
       "      <td>123623</td>\n",
       "      <td>905</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_sum_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>-14.9897</td>\n",
       "      <td>1374.62</td>\n",
       "      <td>-3595.35</td>\n",
       "      <td>-65.6471</td>\n",
       "      <td>-30.1911</td>\n",
       "      <td>-12.4926</td>\n",
       "      <td>137956</td>\n",
       "      <td>123623</td>\n",
       "      <td>123490</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_max_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>36.6086</td>\n",
       "      <td>1348.63</td>\n",
       "      <td>-0.743918</td>\n",
       "      <td>-0.344949</td>\n",
       "      <td>0.13297</td>\n",
       "      <td>1.29638</td>\n",
       "      <td>135766</td>\n",
       "      <td>123623</td>\n",
       "      <td>51993</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_min_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>-0.730221</td>\n",
       "      <td>0.0570227</td>\n",
       "      <td>-0.746908</td>\n",
       "      <td>-0.743902</td>\n",
       "      <td>-0.740446</td>\n",
       "      <td>-0.731881</td>\n",
       "      <td>4.6398</td>\n",
       "      <td>123623</td>\n",
       "      <td>6126</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_mean_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>-0.0728507</td>\n",
       "      <td>27.4661</td>\n",
       "      <td>-0.745643</td>\n",
       "      <td>-0.678985</td>\n",
       "      <td>-0.620652</td>\n",
       "      <td>-0.498502</td>\n",
       "      <td>5426</td>\n",
       "      <td>123623</td>\n",
       "      <td>123486</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_std_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>3.83792</td>\n",
       "      <td>157.59</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0769657</td>\n",
       "      <td>0.16158</td>\n",
       "      <td>0.365731</td>\n",
       "      <td>23023.1</td>\n",
       "      <td>123623</td>\n",
       "      <td>123440</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_var_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>24849.3</td>\n",
       "      <td>2.42659e+06</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00592372</td>\n",
       "      <td>0.026108</td>\n",
       "      <td>0.133759</td>\n",
       "      <td>5.30061e+08</td>\n",
       "      <td>123623</td>\n",
       "      <td>123441</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_sum_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>58.2215</td>\n",
       "      <td>105.198</td>\n",
       "      <td>-16</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>76</td>\n",
       "      <td>6331</td>\n",
       "      <td>123623</td>\n",
       "      <td>959</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_max_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>4.34066</td>\n",
       "      <td>24.6767</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>999</td>\n",
       "      <td>123623</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_min_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>0.053744</td>\n",
       "      <td>0.723538</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>123623</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_mean_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>0.870227</td>\n",
       "      <td>1.38865</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.00190476</td>\n",
       "      <td>1</td>\n",
       "      <td>1.34328</td>\n",
       "      <td>200.8</td>\n",
       "      <td>123623</td>\n",
       "      <td>16374</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_std_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>0.81387</td>\n",
       "      <td>3.77682</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.288675</td>\n",
       "      <td>1.149</td>\n",
       "      <td>446.208</td>\n",
       "      <td>123623</td>\n",
       "      <td>57532</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_var_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>14.9266</td>\n",
       "      <td>941.02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0833333</td>\n",
       "      <td>1.3202</td>\n",
       "      <td>199101</td>\n",
       "      <td>123623</td>\n",
       "      <td>61549</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_sum_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>-403.119</td>\n",
       "      <td>622.098</td>\n",
       "      <td>-22477</td>\n",
       "      <td>-473</td>\n",
       "      <td>-179</td>\n",
       "      <td>-66</td>\n",
       "      <td>-1</td>\n",
       "      <td>123623</td>\n",
       "      <td>3857</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_max_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>-0.161831</td>\n",
       "      <td>0.624408</td>\n",
       "      <td>-11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>123623</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_min_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>-7.88781</td>\n",
       "      <td>3.84051</td>\n",
       "      <td>-13</td>\n",
       "      <td>-12</td>\n",
       "      <td>-8</td>\n",
       "      <td>-4</td>\n",
       "      <td>-1</td>\n",
       "      <td>123623</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_mean_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>-3.84519</td>\n",
       "      <td>2.12383</td>\n",
       "      <td>-12.4615</td>\n",
       "      <td>-5.52613</td>\n",
       "      <td>-3.57325</td>\n",
       "      <td>-2.03226</td>\n",
       "      <td>-0.0227273</td>\n",
       "      <td>123623</td>\n",
       "      <td>40359</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_std_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>2.41353</td>\n",
       "      <td>1.17363</td>\n",
       "      <td>0.129099</td>\n",
       "      <td>1.38034</td>\n",
       "      <td>2.27787</td>\n",
       "      <td>3.44938</td>\n",
       "      <td>6.94742</td>\n",
       "      <td>123623</td>\n",
       "      <td>103125</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_var_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>7.20253</td>\n",
       "      <td>6.13835</td>\n",
       "      <td>0.0166667</td>\n",
       "      <td>1.90533</td>\n",
       "      <td>5.18869</td>\n",
       "      <td>11.8982</td>\n",
       "      <td>48.2667</td>\n",
       "      <td>123623</td>\n",
       "      <td>106727</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amin_old</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>123623</td>\n",
       "      <td>119481</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amax_old</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>123623</td>\n",
       "      <td>117307</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>date</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_ptp_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>244.094</td>\n",
       "      <td>118.189</td>\n",
       "      <td>4</td>\n",
       "      <td>138</td>\n",
       "      <td>233</td>\n",
       "      <td>361</td>\n",
       "      <td>423</td>\n",
       "      <td>123623</td>\n",
       "      <td>420</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id_size_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>89.6464</td>\n",
       "      <td>106.37</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>55</td>\n",
       "      <td>111</td>\n",
       "      <td>5582</td>\n",
       "      <td>123623</td>\n",
       "      <td>943</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_sum_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>25.0781</td>\n",
       "      <td>32.2717</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>31</td>\n",
       "      <td>1304</td>\n",
       "      <td>123623</td>\n",
       "      <td>359</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_mean_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>0.276485</td>\n",
       "      <td>0.139082</td>\n",
       "      <td>0</td>\n",
       "      <td>0.185484</td>\n",
       "      <td>0.269231</td>\n",
       "      <td>0.356026</td>\n",
       "      <td>1</td>\n",
       "      <td>123623</td>\n",
       "      <td>12170</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff_mean_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>11.6402</td>\n",
       "      <td>2.3768</td>\n",
       "      <td>10</td>\n",
       "      <td>10.4444</td>\n",
       "      <td>10.5714</td>\n",
       "      <td>11.5491</td>\n",
       "      <td>23</td>\n",
       "      <td>123623</td>\n",
       "      <td>25231</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inverse_avg_transactions_per_day_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>6.04884</td>\n",
       "      <td>6.77195</td>\n",
       "      <td>0.051057</td>\n",
       "      <td>2</td>\n",
       "      <td>3.77083</td>\n",
       "      <td>7.39286</td>\n",
       "      <td>129.5</td>\n",
       "      <td>123623</td>\n",
       "      <td>36933</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_since_last_transaction_old</th>\n",
       "      <td>123623</td>\n",
       "      <td>360.095</td>\n",
       "      <td>74.0403</td>\n",
       "      <td>314</td>\n",
       "      <td>316</td>\n",
       "      <td>325</td>\n",
       "      <td>372</td>\n",
       "      <td>707</td>\n",
       "      <td>123623</td>\n",
       "      <td>390</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>90 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       count         mean          std  \\\n",
       "first_active_month                       NaN          NaN          NaN   \n",
       "card_id                                  NaN          NaN          NaN   \n",
       "feature_1                             123623      3.10926      1.18911   \n",
       "feature_2                             123623       1.7418     0.749195   \n",
       "feature_3                             123623     0.564377      0.49584   \n",
       "first_active_monthYear                123622      2016.51     0.785047   \n",
       "first_active_monthMonth               123622      7.38177       3.3375   \n",
       "first_active_monthWeek                123622      30.9737      13.8209   \n",
       "first_active_monthDay                 123622            1            0   \n",
       "first_active_monthDayofweek           123622      3.18874       1.8966   \n",
       "first_active_monthDayofyear           123622      194.574        101.7   \n",
       "first_active_monthIs_month_end           NaN          NaN          NaN   \n",
       "first_active_monthIs_month_start         NaN          NaN          NaN   \n",
       "first_active_monthIs_quarter_end         NaN          NaN          NaN   \n",
       "first_active_monthIs_quarter_start       NaN          NaN          NaN   \n",
       "first_active_monthIs_year_end            NaN          NaN          NaN   \n",
       "first_active_monthIs_year_start          NaN          NaN          NaN   \n",
       "first_active_monthElapsed             123623  1.48443e+09  3.95779e+07   \n",
       "transactions_count                    110015      6.75677      6.65022   \n",
       "city_id_nunique                       110015      2.30822      1.65098   \n",
       "merchant_category_id_nunique          110015      5.33326      4.25539   \n",
       "merchant_id_nunique                   110015      6.66368      6.60652   \n",
       "state_id_nunique                      110015      1.58358     0.873022   \n",
       "purchase_Month_nunique                110015      1.65595     0.475061   \n",
       "subsector_id_nunique                  110015       4.4494      3.05787   \n",
       "authorized_flag_mean                  110015            1            0   \n",
       "authorized_flag_sum                   110015      6.75677      6.65022   \n",
       "purchase_amount_sum                   110015     -3.70933      4.50389   \n",
       "purchase_amount_max                   110015    -0.122972      1.76015   \n",
       "purchase_amount_min                   110015    -0.656742     0.912294   \n",
       "...                                      ...          ...          ...   \n",
       "subsector_id_nunique_old              123623      11.4836      5.19046   \n",
       "authorized_flag_mean_old              123623     0.893012     0.114518   \n",
       "authorized_flag_sum_old               123623      81.9209      100.695   \n",
       "purchase_amount_sum_old               123623     -14.9897      1374.62   \n",
       "purchase_amount_max_old               123623      36.6086      1348.63   \n",
       "purchase_amount_min_old               123623    -0.730221    0.0570227   \n",
       "purchase_amount_mean_old              123623   -0.0728507      27.4661   \n",
       "purchase_amount_std_old               123623      3.83792       157.59   \n",
       "purchase_amount_var_old               123623      24849.3  2.42659e+06   \n",
       "installments_sum_old                  123623      58.2215      105.198   \n",
       "installments_max_old                  123623      4.34066      24.6767   \n",
       "installments_min_old                  123623     0.053744     0.723538   \n",
       "installments_mean_old                 123623     0.870227      1.38865   \n",
       "installments_std_old                  123623      0.81387      3.77682   \n",
       "installments_var_old                  123623      14.9266       941.02   \n",
       "month_lag_sum_old                     123623     -403.119      622.098   \n",
       "month_lag_max_old                     123623    -0.161831     0.624408   \n",
       "month_lag_min_old                     123623     -7.88781      3.84051   \n",
       "month_lag_mean_old                    123623     -3.84519      2.12383   \n",
       "month_lag_std_old                     123623      2.41353      1.17363   \n",
       "month_lag_var_old                     123623      7.20253      6.13835   \n",
       "purchase_date_amin_old                   NaN          NaN          NaN   \n",
       "purchase_date_amax_old                   NaN          NaN          NaN   \n",
       "purchase_date_ptp_old                 123623      244.094      118.189   \n",
       "card_id_size_old                      123623      89.6464       106.37   \n",
       "purchased_on_weekend_sum_old          123623      25.0781      32.2717   \n",
       "purchased_on_weekend_mean_old         123623     0.276485     0.139082   \n",
       "month_diff_mean_old                   123623      11.6402       2.3768   \n",
       "inverse_avg_transactions_per_day_old  123623      6.04884      6.77195   \n",
       "days_since_last_transaction_old       123623      360.095      74.0403   \n",
       "\n",
       "                                              min          25%        50%  \\\n",
       "first_active_month                            NaN          NaN        NaN   \n",
       "card_id                                       NaN          NaN        NaN   \n",
       "feature_1                                       1            2          3   \n",
       "feature_2                                       1            1          2   \n",
       "feature_3                                       0            0          1   \n",
       "first_active_monthYear                       2011         2016       2017   \n",
       "first_active_monthMonth                         1            5          8   \n",
       "first_active_monthWeek                          1           22         31   \n",
       "first_active_monthDay                           1            1          1   \n",
       "first_active_monthDayofweek                     0            2          3   \n",
       "first_active_monthDayofyear                     1          121        213   \n",
       "first_active_monthIs_month_end                NaN          NaN        NaN   \n",
       "first_active_monthIs_month_start              NaN          NaN        NaN   \n",
       "first_active_monthIs_quarter_end              NaN          NaN        NaN   \n",
       "first_active_monthIs_quarter_start            NaN          NaN        NaN   \n",
       "first_active_monthIs_year_end                 NaN          NaN        NaN   \n",
       "first_active_monthIs_year_start               NaN          NaN        NaN   \n",
       "first_active_monthElapsed            -9.22337e+09  1.47528e+09  1.491e+09   \n",
       "transactions_count                              1            2          5   \n",
       "city_id_nunique                                 1            1          2   \n",
       "merchant_category_id_nunique                    1            2          4   \n",
       "merchant_id_nunique                             0            2          5   \n",
       "state_id_nunique                                1            1          1   \n",
       "purchase_Month_nunique                          1            1          2   \n",
       "subsector_id_nunique                            1            2          4   \n",
       "authorized_flag_mean                            1            1          1   \n",
       "authorized_flag_sum                             1            2          5   \n",
       "purchase_amount_sum                      -60.7804     -5.21013   -2.66773   \n",
       "purchase_amount_max                     -0.746893    -0.647883  -0.511593   \n",
       "purchase_amount_min                     -0.746893    -0.735638  -0.723617   \n",
       "...                                           ...          ...        ...   \n",
       "subsector_id_nunique_old                        1            8         11   \n",
       "authorized_flag_mean_old                     0.05     0.851852   0.926531   \n",
       "authorized_flag_sum_old                         2           23         48   \n",
       "purchase_amount_sum_old                  -3595.35     -65.6471   -30.1911   \n",
       "purchase_amount_max_old                 -0.743918    -0.344949    0.13297   \n",
       "purchase_amount_min_old                 -0.746908    -0.743902  -0.740446   \n",
       "purchase_amount_mean_old                -0.745643    -0.678985  -0.620652   \n",
       "purchase_amount_std_old                         0    0.0769657    0.16158   \n",
       "purchase_amount_var_old                         0   0.00592372   0.026108   \n",
       "installments_sum_old                          -16            1         16   \n",
       "installments_max_old                            0            1          2   \n",
       "installments_min_old                           -1            0          0   \n",
       "installments_mean_old                        -0.5   0.00190476          1   \n",
       "installments_std_old                            0            0   0.288675   \n",
       "installments_var_old                            0            0  0.0833333   \n",
       "month_lag_sum_old                          -22477         -473       -179   \n",
       "month_lag_max_old                             -11            0          0   \n",
       "month_lag_min_old                             -13          -12         -8   \n",
       "month_lag_mean_old                       -12.4615     -5.52613   -3.57325   \n",
       "month_lag_std_old                        0.129099      1.38034    2.27787   \n",
       "month_lag_var_old                       0.0166667      1.90533    5.18869   \n",
       "purchase_date_amin_old                        NaN          NaN        NaN   \n",
       "purchase_date_amax_old                        NaN          NaN        NaN   \n",
       "purchase_date_ptp_old                           4          138        233   \n",
       "card_id_size_old                                2           27         55   \n",
       "purchased_on_weekend_sum_old                    0            6         14   \n",
       "purchased_on_weekend_mean_old                   0     0.185484   0.269231   \n",
       "month_diff_mean_old                            10      10.4444    10.5714   \n",
       "inverse_avg_transactions_per_day_old     0.051057            2    3.77083   \n",
       "days_since_last_transaction_old               314          316        325   \n",
       "\n",
       "                                              75%          max  counts  \\\n",
       "first_active_month                            NaN          NaN  123622   \n",
       "card_id                                       NaN          NaN  123623   \n",
       "feature_1                                       4            5  123623   \n",
       "feature_2                                       2            3  123623   \n",
       "feature_3                                       1            1  123623   \n",
       "first_active_monthYear                       2017         2018  123622   \n",
       "first_active_monthMonth                        10           12  123622   \n",
       "first_active_monthWeek                         44           53  123622   \n",
       "first_active_monthDay                           1            1  123622   \n",
       "first_active_monthDayofweek                     5            6  123622   \n",
       "first_active_monthDayofyear                   274          336  123622   \n",
       "first_active_monthIs_month_end                NaN          NaN  123623   \n",
       "first_active_monthIs_month_start              NaN          NaN  123623   \n",
       "first_active_monthIs_quarter_end              NaN          NaN  123623   \n",
       "first_active_monthIs_quarter_start            NaN          NaN  123623   \n",
       "first_active_monthIs_year_end                 NaN          NaN  123623   \n",
       "first_active_monthIs_year_start               NaN          NaN  123623   \n",
       "first_active_monthElapsed             1.50422e+09  1.51476e+09  123623   \n",
       "transactions_count                              9          104  110015   \n",
       "city_id_nunique                                 3           32  110015   \n",
       "merchant_category_id_nunique                    7           46  110015   \n",
       "merchant_id_nunique                             9          103  110015   \n",
       "state_id_nunique                                2           15  110015   \n",
       "purchase_Month_nunique                          2            2  110015   \n",
       "subsector_id_nunique                            6           23  110015   \n",
       "authorized_flag_mean                            1            1  110015   \n",
       "authorized_flag_sum                             9          104  110015   \n",
       "purchase_amount_sum                       -1.2066      263.157  110015   \n",
       "purchase_amount_max                      -0.17593      263.157  110015   \n",
       "purchase_amount_min                      -0.69161      263.157  110015   \n",
       "...                                           ...          ...     ...   \n",
       "subsector_id_nunique_old                       15           34  123623   \n",
       "authorized_flag_mean_old                 0.972603            1  123623   \n",
       "authorized_flag_sum_old                       100         4122  123623   \n",
       "purchase_amount_sum_old                  -12.4926       137956  123623   \n",
       "purchase_amount_max_old                   1.29638       135766  123623   \n",
       "purchase_amount_min_old                 -0.731881       4.6398  123623   \n",
       "purchase_amount_mean_old                -0.498502         5426  123623   \n",
       "purchase_amount_std_old                  0.365731      23023.1  123623   \n",
       "purchase_amount_var_old                  0.133759  5.30061e+08  123623   \n",
       "installments_sum_old                           76         6331  123623   \n",
       "installments_max_old                            6          999  123623   \n",
       "installments_min_old                            1           10  123623   \n",
       "installments_mean_old                     1.34328        200.8  123623   \n",
       "installments_std_old                        1.149      446.208  123623   \n",
       "installments_var_old                       1.3202       199101  123623   \n",
       "month_lag_sum_old                             -66           -1  123623   \n",
       "month_lag_max_old                               0            0  123623   \n",
       "month_lag_min_old                              -4           -1  123623   \n",
       "month_lag_mean_old                       -2.03226   -0.0227273  123623   \n",
       "month_lag_std_old                         3.44938      6.94742  123623   \n",
       "month_lag_var_old                         11.8982      48.2667  123623   \n",
       "purchase_date_amin_old                        NaN          NaN  123623   \n",
       "purchase_date_amax_old                        NaN          NaN  123623   \n",
       "purchase_date_ptp_old                         361          423  123623   \n",
       "card_id_size_old                              111         5582  123623   \n",
       "purchased_on_weekend_sum_old                   31         1304  123623   \n",
       "purchased_on_weekend_mean_old            0.356026            1  123623   \n",
       "month_diff_mean_old                       11.5491           23  123623   \n",
       "inverse_avg_transactions_per_day_old      7.39286        129.5  123623   \n",
       "days_since_last_transaction_old               372          707  123623   \n",
       "\n",
       "                                     uniques missing missing_perc     types  \n",
       "first_active_month                        75       1        0.00%      date  \n",
       "card_id                               123623       0           0%    unique  \n",
       "feature_1                                  5       0           0%   numeric  \n",
       "feature_2                                  3       0           0%   numeric  \n",
       "feature_3                                  2       0           0%      bool  \n",
       "first_active_monthYear                     8       1        0.00%   numeric  \n",
       "first_active_monthMonth                   12       1        0.00%   numeric  \n",
       "first_active_monthWeek                    21       1        0.00%   numeric  \n",
       "first_active_monthDay                      1       1        0.00%  constant  \n",
       "first_active_monthDayofweek                7       1        0.00%   numeric  \n",
       "first_active_monthDayofyear               22       1        0.00%   numeric  \n",
       "first_active_monthIs_month_end             1       0           0%  constant  \n",
       "first_active_monthIs_month_start           2       0           0%      bool  \n",
       "first_active_monthIs_quarter_end           1       0           0%  constant  \n",
       "first_active_monthIs_quarter_start         2       0           0%      bool  \n",
       "first_active_monthIs_year_end              1       0           0%  constant  \n",
       "first_active_monthIs_year_start            2       0           0%      bool  \n",
       "first_active_monthElapsed                 76       0           0%   numeric  \n",
       "transactions_count                        80   13608       11.01%   numeric  \n",
       "city_id_nunique                           24   13608       11.01%   numeric  \n",
       "merchant_category_id_nunique              39   13608       11.01%   numeric  \n",
       "merchant_id_nunique                       79   13608       11.01%   numeric  \n",
       "state_id_nunique                          14   13608       11.01%   numeric  \n",
       "purchase_Month_nunique                     2   13608       11.01%      bool  \n",
       "subsector_id_nunique                      23   13608       11.01%   numeric  \n",
       "authorized_flag_mean                       1   13608       11.01%  constant  \n",
       "authorized_flag_sum                       80   13608       11.01%   numeric  \n",
       "purchase_amount_sum                    94244   13608       11.01%   numeric  \n",
       "purchase_amount_max                    32279   13608       11.01%   numeric  \n",
       "purchase_amount_min                    12517   13608       11.01%   numeric  \n",
       "...                                      ...     ...          ...       ...  \n",
       "subsector_id_nunique_old                  34       0           0%   numeric  \n",
       "authorized_flag_mean_old                9131       0           0%   numeric  \n",
       "authorized_flag_sum_old                  905       0           0%   numeric  \n",
       "purchase_amount_sum_old               123490       0           0%   numeric  \n",
       "purchase_amount_max_old                51993       0           0%   numeric  \n",
       "purchase_amount_min_old                 6126       0           0%   numeric  \n",
       "purchase_amount_mean_old              123486       0           0%   numeric  \n",
       "purchase_amount_std_old               123440       0           0%   numeric  \n",
       "purchase_amount_var_old               123441       0           0%   numeric  \n",
       "installments_sum_old                     959       0           0%   numeric  \n",
       "installments_max_old                      14       0           0%   numeric  \n",
       "installments_min_old                       9       0           0%   numeric  \n",
       "installments_mean_old                  16374       0           0%   numeric  \n",
       "installments_std_old                   57532       0           0%   numeric  \n",
       "installments_var_old                   61549       0           0%   numeric  \n",
       "month_lag_sum_old                       3857       0           0%   numeric  \n",
       "month_lag_max_old                         12       0           0%   numeric  \n",
       "month_lag_min_old                         13       0           0%   numeric  \n",
       "month_lag_mean_old                     40359       0           0%   numeric  \n",
       "month_lag_std_old                     103125       0           0%   numeric  \n",
       "month_lag_var_old                     106727       0           0%   numeric  \n",
       "purchase_date_amin_old                119481       0           0%      date  \n",
       "purchase_date_amax_old                117307       0           0%      date  \n",
       "purchase_date_ptp_old                    420       0           0%   numeric  \n",
       "card_id_size_old                         943       0           0%   numeric  \n",
       "purchased_on_weekend_sum_old             359       0           0%   numeric  \n",
       "purchased_on_weekend_mean_old          12170       0           0%   numeric  \n",
       "month_diff_mean_old                    25231       0           0%   numeric  \n",
       "inverse_avg_transactions_per_day_old   36933       0           0%   numeric  \n",
       "days_since_last_transaction_old          390       0           0%   numeric  \n",
       "\n",
       "[90 rows x 13 columns]"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DataFrameSummary(test_df2).summary().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df2.to_feather('train_df2')\n",
    "test_df2.to_feather('test_df2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['first_active_month', 'card_id', 'feature_1', 'feature_2', 'feature_3', 'target',\n",
       "       'first_active_monthYear', 'first_active_monthMonth', 'first_active_monthWeek',\n",
       "       'first_active_monthDay', 'first_active_monthDayofweek', 'first_active_monthDayofyear',\n",
       "       'first_active_monthIs_month_end', 'first_active_monthIs_month_start',\n",
       "       'first_active_monthIs_quarter_end', 'first_active_monthIs_quarter_start',\n",
       "       'first_active_monthIs_year_end', 'first_active_monthIs_year_start', 'first_active_monthElapsed',\n",
       "       'transactions_count', 'city_id_nunique', 'merchant_category_id_nunique', 'merchant_id_nunique',\n",
       "       'state_id_nunique', 'purchase_Month_nunique', 'subsector_id_nunique', 'authorized_flag_mean',\n",
       "       'authorized_flag_sum', 'purchase_amount_sum', 'purchase_amount_max', 'purchase_amount_min',\n",
       "       'purchase_amount_mean', 'purchase_amount_std', 'purchase_amount_var', 'installments_sum',\n",
       "       'installments_max', 'installments_min', 'installments_mean', 'installments_std', 'installments_var',\n",
       "       'month_lag_sum', 'month_lag_max', 'month_lag_min', 'month_lag_mean', 'month_lag_std', 'month_lag_var',\n",
       "       'purchase_date_amin', 'purchase_date_amax', 'purchase_date_ptp', 'card_id_size',\n",
       "       'purchased_on_weekend_sum', 'purchased_on_weekend_mean', 'month_diff_mean', 'category_1_sum',\n",
       "       'category_1_mean', 'inverse_avg_transactions_per_day', 'days_since_last_transaction',\n",
       "       'transactions_count_old', 'city_id_nunique_old', 'merchant_category_id_nunique_old',\n",
       "       'merchant_id_nunique_old', 'state_id_nunique_old', 'purchase_Month_nunique_old',\n",
       "       'subsector_id_nunique_old', 'authorized_flag_mean_old', 'authorized_flag_sum_old',\n",
       "       'purchase_amount_sum_old', 'purchase_amount_max_old', 'purchase_amount_min_old',\n",
       "       'purchase_amount_mean_old', 'purchase_amount_std_old', 'purchase_amount_var_old',\n",
       "       'installments_sum_old', 'installments_max_old', 'installments_min_old', 'installments_mean_old',\n",
       "       'installments_std_old', 'installments_var_old', 'month_lag_sum_old', 'month_lag_max_old',\n",
       "       'month_lag_min_old', 'month_lag_mean_old', 'month_lag_std_old', 'month_lag_var_old',\n",
       "       'purchase_date_amin_old', 'purchase_date_amax_old', 'purchase_date_ptp_old', 'card_id_size_old',\n",
       "       'purchased_on_weekend_sum_old', 'purchased_on_weekend_mean_old', 'month_diff_mean_old',\n",
       "       'category_1_sum_old', 'category_1_mean_old', 'inverse_avg_transactions_per_day_old',\n",
       "       'days_since_last_transaction_old'], dtype=object)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df2.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "for df in [train_df2, test_df2]:\n",
    "    df['elapsed_time'] = (datetime.datetime.today() - df['first_active_month']).dt.days\n",
    "    df['card_id_total'] = df['card_id_size']+df['card_id_size_old']\n",
    "    df['purchase_amount_total'] = df['purchase_amount_sum']+df['purchase_amount_sum_old']\n",
    "    df['installments_total'] = df['installments_sum'] + df['installments_sum_old']\n",
    "    df['hist_first_buy'] = (df['purchase_date_amin_old'] - df['first_active_month']).dt.days\n",
    "    df['new_first_buy'] = (df['purchase_date_amin'] - df['first_active_month']).dt.days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(201917, 101)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df2.to_feather('train_df2')\n",
    "test_df2.to_feather('test_df2')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df2['outliers'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    199710\n",
       "1      2207\n",
       "Name: outliers, dtype: int64"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df2.loc[train_df2['target'] < -30, 'outliers'] = 1\n",
    "train_df2['outliers'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Target encoding for outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "feature_1\n",
       "1    0.008058\n",
       "2    0.010610\n",
       "3    0.010479\n",
       "4    0.010712\n",
       "5    0.013145\n",
       "Name: outliers, dtype: float64"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df2.groupby(['feature_1'])['outliers'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in ['feature_1','feature_2','feature_3']:\n",
    "    order_label = train_df2.groupby([f])['outliers'].mean()\n",
    "    train_df2[f] = train_df2[f].map(order_label)\n",
    "    test_df2[f] = test_df2[f].map(order_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df2.to_feather('train_df2')\n",
    "test_df2.to_feather('test_df2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}