Master Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "aisles = pd.read_csv('data/aisles.csv')\n",
    "departments = pd.read_csv('data/departments.csv')\n",
    "order_products_prior = pd.read_csv('data/order_products__prior.csv')\n",
    "order_products_train = pd.read_csv('data/order_products__train.csv')\n",
    "products= pd.read_csv('data/products.csv')\n",
    "sample_submission = pd.read_csv('data/sample_submission.csv')\n",
    "orders = pd.read_csv('data/orders.csv')\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "aisles: (134, 2)\n",
      "departments: (21, 2)\n",
      "order_products_prior: (32434489, 4)\n",
      "order_products_train: (1384617, 4)\n",
      "products: (49688, 4)\n",
      "sample_submission: (75000, 2)\n",
      "orders: (3421083, 7)\n"
     ]
    }
   ],
   "source": [
    "#princt shapes for all\n",
    "print('aisles:',aisles.shape)\n",
    "print('departments:',departments.shape)\n",
    "print('order_products_prior:',order_products_prior.shape)\n",
    "print('order_products_train:',order_products_train.shape)\n",
    "print('products:',products.shape)\n",
    "print('sample_submission:',sample_submission.shape)\n",
    "print('orders:',orders.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>aisle</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>prepared soups salads</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>specialty cheeses</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>energy granola bars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>instant foods</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>marinades meat preparation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   aisle_id                       aisle\n",
       "0         1       prepared soups salads\n",
       "1         2           specialty cheeses\n",
       "2         3         energy granola bars\n",
       "3         4               instant foods\n",
       "4         5  marinades meat preparation"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aisles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>department_id</th>\n",
       "      <th>department</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>frozen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>bakery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>produce</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>alcohol</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   department_id department\n",
       "0              1     frozen\n",
       "1              2      other\n",
       "2              3     bakery\n",
       "3              4    produce\n",
       "4              5    alcohol"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "departments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>add_to_cart_order</th>\n",
       "      <th>reordered</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>33120</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>28985</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>9327</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>45918</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>30035</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id  product_id  add_to_cart_order  reordered\n",
       "0         2       33120                  1          1\n",
       "1         2       28985                  2          1\n",
       "2         2        9327                  3          0\n",
       "3         2       45918                  4          1\n",
       "4         2       30035                  5          0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "order_products_prior.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>add_to_cart_order</th>\n",
       "      <th>reordered</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>49302</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>11109</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>10246</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>49683</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>43633</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id  product_id  add_to_cart_order  reordered\n",
       "0         1       49302                  1          1\n",
       "1         1       11109                  2          1\n",
       "2         1       10246                  3          0\n",
       "3         1       49683                  4          0\n",
       "4         1       43633                  5          1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "order_products_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product_id</th>\n",
       "      <th>product_name</th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>department_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Chocolate Sandwich Cookies</td>\n",
       "      <td>61</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>All-Seasons Salt</td>\n",
       "      <td>104</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Robust Golden Unsweetened Oolong Tea</td>\n",
       "      <td>94</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Smart Ones Classic Favorites Mini Rigatoni Wit...</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Green Chile Anytime Sauce</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   product_id                                       product_name  aisle_id  \\\n",
       "0           1                         Chocolate Sandwich Cookies        61   \n",
       "1           2                                   All-Seasons Salt       104   \n",
       "2           3               Robust Golden Unsweetened Oolong Tea        94   \n",
       "3           4  Smart Ones Classic Favorites Mini Rigatoni Wit...        38   \n",
       "4           5                          Green Chile Anytime Sauce         5   \n",
       "\n",
       "   department_id  \n",
       "0             19  \n",
       "1             13  \n",
       "2              7  \n",
       "3              1  \n",
       "4             13  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "products.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>products</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17</td>\n",
       "      <td>39276 29259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34</td>\n",
       "      <td>39276 29259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>137</td>\n",
       "      <td>39276 29259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>182</td>\n",
       "      <td>39276 29259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>257</td>\n",
       "      <td>39276 29259</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id     products\n",
       "0        17  39276 29259\n",
       "1        34  39276 29259\n",
       "2       137  39276 29259\n",
       "3       182  39276 29259\n",
       "4       257  39276 29259"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>eval_set</th>\n",
       "      <th>order_number</th>\n",
       "      <th>order_dow</th>\n",
       "      <th>order_hour_of_day</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2539329</td>\n",
       "      <td>1</td>\n",
       "      <td>prior</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2398795</td>\n",
       "      <td>1</td>\n",
       "      <td>prior</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>473747</td>\n",
       "      <td>1</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2254736</td>\n",
       "      <td>1</td>\n",
       "      <td>prior</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>431534</td>\n",
       "      <td>1</td>\n",
       "      <td>prior</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>15</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id  user_id eval_set  order_number  order_dow  order_hour_of_day  \\\n",
       "0   2539329        1    prior             1          2                  8   \n",
       "1   2398795        1    prior             2          3                  7   \n",
       "2    473747        1    prior             3          3                 12   \n",
       "3   2254736        1    prior             4          4                  7   \n",
       "4    431534        1    prior             5          4                 15   \n",
       "\n",
       "   days_since_prior_order  \n",
       "0                     NaN  \n",
       "1                    15.0  \n",
       "2                    21.0  \n",
       "3                    29.0  \n",
       "4                    28.0  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orders.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3346083, 7)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "#check unique values of eval_set\n",
    "orders['eval_set'].unique()\n",
    "\n",
    "#droopping the eval_set = test as we do not have target variable(reorder) for this\n",
    "orders = orders[orders['eval_set'] != 'test']\n",
    "orders.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/1p/q_9hhzqs0yddpgkxqm_hvtbc0000gn/T/ipykernel_66618/4124274290.py:2: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  data = order_products_prior.append(order_products_train)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(33819106, 4)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#append order_products_prior and order_products_train\n",
    "data = order_products_prior.append(order_products_train)\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33819106, 10)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#left join order_products with orders\n",
    "data = pd.merge(data, orders, on='order_id', how='left')\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(33819106, 13)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>add_to_cart_order</th>\n",
       "      <th>reordered</th>\n",
       "      <th>user_id</th>\n",
       "      <th>eval_set</th>\n",
       "      <th>order_number</th>\n",
       "      <th>order_dow</th>\n",
       "      <th>order_hour_of_day</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "      <th>product_name</th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>department_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>33120</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Organic Egg Whites</td>\n",
       "      <td>86</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>28985</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Michigan Organic Kale</td>\n",
       "      <td>83</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>9327</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Garlic Powder</td>\n",
       "      <td>104</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>45918</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Coconut Butter</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>30035</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Natural Sweetener</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id  product_id  add_to_cart_order  reordered  user_id eval_set  \\\n",
       "0         2       33120                  1          1   202279    prior   \n",
       "1         2       28985                  2          1   202279    prior   \n",
       "2         2        9327                  3          0   202279    prior   \n",
       "3         2       45918                  4          1   202279    prior   \n",
       "4         2       30035                  5          0   202279    prior   \n",
       "\n",
       "   order_number  order_dow  order_hour_of_day  days_since_prior_order  \\\n",
       "0             3          5                  9                     8.0   \n",
       "1             3          5                  9                     8.0   \n",
       "2             3          5                  9                     8.0   \n",
       "3             3          5                  9                     8.0   \n",
       "4             3          5                  9                     8.0   \n",
       "\n",
       "            product_name  aisle_id  department_id  \n",
       "0     Organic Egg Whites        86             16  \n",
       "1  Michigan Organic Kale        83              4  \n",
       "2          Garlic Powder       104             13  \n",
       "3         Coconut Butter        19             13  \n",
       "4      Natural Sweetener        17             13  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.merge(data, products, on='product_id', how='left')\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(33819106, 14)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>add_to_cart_order</th>\n",
       "      <th>reordered</th>\n",
       "      <th>user_id</th>\n",
       "      <th>eval_set</th>\n",
       "      <th>order_number</th>\n",
       "      <th>order_dow</th>\n",
       "      <th>order_hour_of_day</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "      <th>product_name</th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>department_id</th>\n",
       "      <th>aisle</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>33120</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Organic Egg Whites</td>\n",
       "      <td>86</td>\n",
       "      <td>16</td>\n",
       "      <td>eggs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>28985</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Michigan Organic Kale</td>\n",
       "      <td>83</td>\n",
       "      <td>4</td>\n",
       "      <td>fresh vegetables</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>9327</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Garlic Powder</td>\n",
       "      <td>104</td>\n",
       "      <td>13</td>\n",
       "      <td>spices seasonings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>45918</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Coconut Butter</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "      <td>oils vinegars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>30035</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Natural Sweetener</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>baking ingredients</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id  product_id  add_to_cart_order  reordered  user_id eval_set  \\\n",
       "0         2       33120                  1          1   202279    prior   \n",
       "1         2       28985                  2          1   202279    prior   \n",
       "2         2        9327                  3          0   202279    prior   \n",
       "3         2       45918                  4          1   202279    prior   \n",
       "4         2       30035                  5          0   202279    prior   \n",
       "\n",
       "   order_number  order_dow  order_hour_of_day  days_since_prior_order  \\\n",
       "0             3          5                  9                     8.0   \n",
       "1             3          5                  9                     8.0   \n",
       "2             3          5                  9                     8.0   \n",
       "3             3          5                  9                     8.0   \n",
       "4             3          5                  9                     8.0   \n",
       "\n",
       "            product_name  aisle_id  department_id               aisle  \n",
       "0     Organic Egg Whites        86             16                eggs  \n",
       "1  Michigan Organic Kale        83              4    fresh vegetables  \n",
       "2          Garlic Powder       104             13   spices seasonings  \n",
       "3         Coconut Butter        19             13       oils vinegars  \n",
       "4      Natural Sweetener        17             13  baking ingredients  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#left join data with aisles\n",
    "data = pd.merge(data, aisles, on='aisle_id', how='left')\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(33819106, 15)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>add_to_cart_order</th>\n",
       "      <th>reordered</th>\n",
       "      <th>user_id</th>\n",
       "      <th>eval_set</th>\n",
       "      <th>order_number</th>\n",
       "      <th>order_dow</th>\n",
       "      <th>order_hour_of_day</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "      <th>product_name</th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>department_id</th>\n",
       "      <th>aisle</th>\n",
       "      <th>department</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>33120</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Organic Egg Whites</td>\n",
       "      <td>86</td>\n",
       "      <td>16</td>\n",
       "      <td>eggs</td>\n",
       "      <td>dairy eggs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>28985</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Michigan Organic Kale</td>\n",
       "      <td>83</td>\n",
       "      <td>4</td>\n",
       "      <td>fresh vegetables</td>\n",
       "      <td>produce</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>9327</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Garlic Powder</td>\n",
       "      <td>104</td>\n",
       "      <td>13</td>\n",
       "      <td>spices seasonings</td>\n",
       "      <td>pantry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>45918</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Coconut Butter</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "      <td>oils vinegars</td>\n",
       "      <td>pantry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>30035</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Natural Sweetener</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>baking ingredients</td>\n",
       "      <td>pantry</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id  product_id  add_to_cart_order  reordered  user_id eval_set  \\\n",
       "0         2       33120                  1          1   202279    prior   \n",
       "1         2       28985                  2          1   202279    prior   \n",
       "2         2        9327                  3          0   202279    prior   \n",
       "3         2       45918                  4          1   202279    prior   \n",
       "4         2       30035                  5          0   202279    prior   \n",
       "\n",
       "   order_number  order_dow  order_hour_of_day  days_since_prior_order  \\\n",
       "0             3          5                  9                     8.0   \n",
       "1             3          5                  9                     8.0   \n",
       "2             3          5                  9                     8.0   \n",
       "3             3          5                  9                     8.0   \n",
       "4             3          5                  9                     8.0   \n",
       "\n",
       "            product_name  aisle_id  department_id               aisle  \\\n",
       "0     Organic Egg Whites        86             16                eggs   \n",
       "1  Michigan Organic Kale        83              4    fresh vegetables   \n",
       "2          Garlic Powder       104             13   spices seasonings   \n",
       "3         Coconut Butter        19             13       oils vinegars   \n",
       "4      Natural Sweetener        17             13  baking ingredients   \n",
       "\n",
       "   department  \n",
       "0  dairy eggs  \n",
       "1     produce  \n",
       "2      pantry  \n",
       "3      pantry  \n",
       "4      pantry  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#left join data with departments\n",
    "data = pd.merge(data, departments, on='department_id', how='left')\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>add_to_cart_order</th>\n",
       "      <th>user_id</th>\n",
       "      <th>eval_set</th>\n",
       "      <th>order_number</th>\n",
       "      <th>order_dow</th>\n",
       "      <th>order_hour_of_day</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "      <th>product_name</th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>department_id</th>\n",
       "      <th>aisle</th>\n",
       "      <th>department</th>\n",
       "      <th>reordered</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>33120</td>\n",
       "      <td>1</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Organic Egg Whites</td>\n",
       "      <td>86</td>\n",
       "      <td>16</td>\n",
       "      <td>eggs</td>\n",
       "      <td>dairy eggs</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>28985</td>\n",
       "      <td>2</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Michigan Organic Kale</td>\n",
       "      <td>83</td>\n",
       "      <td>4</td>\n",
       "      <td>fresh vegetables</td>\n",
       "      <td>produce</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>9327</td>\n",
       "      <td>3</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Garlic Powder</td>\n",
       "      <td>104</td>\n",
       "      <td>13</td>\n",
       "      <td>spices seasonings</td>\n",
       "      <td>pantry</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>45918</td>\n",
       "      <td>4</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Coconut Butter</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "      <td>oils vinegars</td>\n",
       "      <td>pantry</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>30035</td>\n",
       "      <td>5</td>\n",
       "      <td>202279</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Natural Sweetener</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>baking ingredients</td>\n",
       "      <td>pantry</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id  product_id  add_to_cart_order  user_id eval_set  order_number  \\\n",
       "0         2       33120                  1   202279    prior             3   \n",
       "1         2       28985                  2   202279    prior             3   \n",
       "2         2        9327                  3   202279    prior             3   \n",
       "3         2       45918                  4   202279    prior             3   \n",
       "4         2       30035                  5   202279    prior             3   \n",
       "\n",
       "   order_dow  order_hour_of_day  days_since_prior_order  \\\n",
       "0          5                  9                     8.0   \n",
       "1          5                  9                     8.0   \n",
       "2          5                  9                     8.0   \n",
       "3          5                  9                     8.0   \n",
       "4          5                  9                     8.0   \n",
       "\n",
       "            product_name  aisle_id  department_id               aisle  \\\n",
       "0     Organic Egg Whites        86             16                eggs   \n",
       "1  Michigan Organic Kale        83              4    fresh vegetables   \n",
       "2          Garlic Powder       104             13   spices seasonings   \n",
       "3         Coconut Butter        19             13       oils vinegars   \n",
       "4      Natural Sweetener        17             13  baking ingredients   \n",
       "\n",
       "   department  reordered  \n",
       "0  dairy eggs          1  \n",
       "1     produce          1  \n",
       "2      pantry          0  \n",
       "3      pantry          1  \n",
       "4      pantry          0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#moving target column to the end\n",
    "data = data[[c for c in data if c not in ['reordered']] + ['reordered']]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sorting data by user_id and droping the data where user_id > 25 percentile\n",
    "data2 = data.sort_values('user_id')\n",
    "data2 = data[data['user_id'] <= data['user_id'].quantile(0.01)]\n",
    "data2.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving the data as master_data2.csv\n",
    "data2.to_csv('saved_datasets/master_data2.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}