submission_prediction_output (1).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IronHacks Submission Notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Background\n",
    "---\n",
    "\n",
    "The method I am using for this submission is to take the average of the predition from random forest model with the prediction from moving averge model with period equal to 3 inorder to remove outliers and improve the accuracy of the prediction."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "%logstop\n",
    "%logstart -t -r -q ipython_command_log.py global\n",
    "\n",
    "#- IRONHACKS RESEARCH TRACKING CODE\n",
    "#----------------------------------\n",
    "# The following code is used to help our research team understand how you \n",
    "# our notebook environment. We do not collect any personal information with\n",
    "# the following code, it is used to measure when and how often you work on\n",
    "# your submission files.\n",
    "\n",
    "import os\n",
    "from datetime import datetime\n",
    "import IPython.core.history as history\n",
    "\n",
    "ha = history.HistoryAccessor()\n",
    "ha_tail = ha.get_tail(1)\n",
    "ha_cmd = next(ha_tail)\n",
    "session_id = str(ha_cmd[0])\n",
    "command_id = str(ha_cmd[1])\n",
    "timestamp = datetime.utcnow().isoformat()\n",
    "history_line = ','.join([session_id, command_id, timestamp]) + '\\n'\n",
    "logfile = open(os.environ['HOME']+'/ipython_session_log.csv', 'a')\n",
    "logfile.write(history_line)\n",
    "logfile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "#- INSTALL ADDITIONAL LIBRARIES IF REQUIRED\n",
    "#------------------------------------------\n",
    "# This is normally not required. The hub environment comes preinstaled with \n",
    "# many packages that you can already use without setup. In case there is some\n",
    "# other library you would like to use that isn't on the list you run this command\n",
    "# once to install them.  If it is already installed this command has no effect.\n",
    "!python3 -m pip install google.cloud\n",
    "!python3 -m pip install numpy\n",
    "!python3 -m pip install pyarrow\n",
    "!python3 -m pip install pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#- IMPORT THE LIBRARIES YOU WILL USE\n",
    "#------------------------------------------\n",
    "# You only need to import packages one time per notebook session. To keep your\n",
    "# notebook clean and organized you can handle all imports at the top of your file.\n",
    "# The following are included for example purposed, feel free to modify or delete \n",
    "# anything in this section.\n",
    "\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from google.cloud import bigquery\n",
    "from google.oauth2 import service_account\n",
    "from google.cloud.bigquery import magics\n",
    "magics.context.use_bqstorage_api = True\n",
    "import pyarrow\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from matplotlib import pyplot as plt\n",
    "import random\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "np.set_printoptions(suppress=True, linewidth=200, edgeitems=100)\n",
    "\n",
    "# Pipeline and column transformer\n",
    "from sklearn.pipeline import Pipeline, make_pipeline\n",
    "from sklearn.compose import ColumnTransformer, make_column_transformer\n",
    "\n",
    "# Data transformers\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler, MinMaxScaler, label_binarize\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n",
    "\n",
    "# Data splitter and model evaluator\n",
    "from sklearn.model_selection import train_test_split, validation_curve, learning_curve, GridSearchCV\n",
    "\n",
    "# Learning models\n",
    "from sklearn.linear_model import LinearRegression, Ridge, Lasso, LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor, plot_tree\n",
    "from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n",
    "from sklearn.ensemble import BaggingClassifier, BaggingRegressor, GradientBoostingClassifier, GradientBoostingRegressor\n",
    "#from xgboost import XGBClassifier, XGBRegressor  # Need to install\n",
    "\n",
    "# Performance metrics\n",
    "from sklearn.metrics import log_loss, roc_auc_score\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CONFIGURE THE BIGQUERY SETTINGS\n",
    "\n",
    "BIGQUERY_PROJECT = 'ironhacks-covid19-data'\n",
    "BIGQUERY_KEYPATH = 'service-account.json'\n",
    "\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = BIGQUERY_KEYPATH\n",
    "bigquery_client = bigquery.Client(project=BIGQUERY_PROJECT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classes and Functions\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#- DEFINE YOUR CLASSES AND FUNCTIONS \n",
    "#-----------------------------------\n",
    "# This is not required, but is helpful in keeping your notebook organized. \n",
    "# You can use the following cell or several cells to define your functions\n",
    "# and classes to keep them separate from your analysis or results code.\n",
    "# In general it useful to define your methods in a separate cell from where\n",
    "# it is run.\n",
    "\n",
    "def example_function():\n",
    "    print('Hello World')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function is to get the data I need from the big query\n",
    "\n",
    "def get_data():\n",
    "    query = \"\"\"\n",
    "    SELECT poi_id, top_category, week_number, raw_visit_counts, postal_code\n",
    "    FROM ironhacks_covid19_competition.weekly_patterns\n",
    "    \"\"\"\n",
    "    query_job = bigquery_client.query(query)\n",
    "    wp = query_job.to_dataframe()\n",
    "\n",
    "    query = \"\"\"\n",
    "    SELECT week_number, cases\n",
    "    FROM ironhacks_covid19_competition.covid19_cases\n",
    "    \"\"\"\n",
    "    query_job = bigquery_client.query(query)\n",
    "    covid = query_job.to_dataframe()\n",
    "    return wp, covid\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#shifting the covid_cases up one and two weeks to decrease the delay in collecting data and people's reaction towards the coivd cases\n",
    "\n",
    "def add_covid_cases(covid, wp):\n",
    "    nc = covid.copy()\n",
    "    nc['cases_shift1'] = nc['cases'].shift(1)\n",
    "    nc['cases_shift2'] = nc['cases'].shift(2)\n",
    "    del nc['cases']\n",
    "    nc = nc.fillna(7)\n",
    "    new = wp.merge(nc, on = 'week_number')\n",
    "    return new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing the format of the table to having all the categories, poi_id, and postal_code as column headders\n",
    "# The fillers will be containing 0 or 1 depending on the values in week_number and raw_visit_counts\n",
    "\n",
    "def change_format(new):\n",
    "    dummy = pd.get_dummies(new['top_category'])\n",
    "    table = new.join(dummy)\n",
    "    dummy = pd.get_dummies(new['poi_id'])\n",
    "    table = table.join(dummy)\n",
    "    dummy = pd.get_dummies(new['postal_code'])\n",
    "    table = table.join(dummy)\n",
    "    del table['top_category']\n",
    "    del table['poi_id']\n",
    "    del table['postal_code']\n",
    "    return table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here I'm making my predictinon table which contains all the known informations for week 44 except for raw_visit_counts\n",
    "\n",
    "def prediction_data(new):\n",
    "    filt_wp = new[new['week_number'] == 43]\n",
    "    dummy = pd.get_dummies(filt_wp['top_category'])\n",
    "    table2 = filt_wp.join(dummy)\n",
    "    dummy = pd.get_dummies(filt_wp['poi_id'])\n",
    "    table2 = table2.join(dummy)\n",
    "    dummy = pd.get_dummies(filt_wp['postal_code'])\n",
    "    table2 = table2.join(dummy)\n",
    "    del table2['top_category']\n",
    "    del table2['raw_visit_counts']\n",
    "    del table2['postal_code']\n",
    "\n",
    "    table2['week_number'] = table2['week_number'].replace(43, 44)\n",
    "    table2['cases_shift1'] = table2['cases_shift1'].replace(25872, 29641)\n",
    "    table2['cases_shift2'] = table2['cases_shift2'].replace(23218, 25872)\n",
    "    return table2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Making a decision tree model based on the table we've given to it\n",
    "# I set raw_visit_counts as the Y value and everything else is X\n",
    "# Also splited the data randomly into 80% for training set and 20% for testing\n",
    "# Prints out the training score and testing score\n",
    "\n",
    "\n",
    "def decision_tree(table):\n",
    "    df = table\n",
    "    X = df.drop(['raw_visit_counts'], axis=1)\n",
    "    y = df['raw_visit_counts']\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, \n",
    "                                                        test_size=0.3, \n",
    "                                                        random_state=0)\n",
    "\n",
    "    decision_tree_model = DecisionTreeRegressor(random_state=0)\n",
    "    decision_tree_model.fit(X_train, y_train)\n",
    "    print('Decision Tree Training score:', decision_tree_model.score(X_train,y_train))\n",
    "    print('Decision Tree Test score:    ', decision_tree_model.score(X_test,y_test),'\\n')\n",
    "    return decision_tree_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# same as decision_tree model but used random_forest model\n",
    "def random_forest(table):\n",
    "    df = table\n",
    "    X = df.drop(['raw_visit_counts'], axis=1)\n",
    "    y = df['raw_visit_counts']\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, \n",
    "                                                        test_size=0.3, \n",
    "                                                        random_state=0)\n",
    "    model = RandomForestRegressor(random_state=0)\n",
    "    model.fit(X_train, y_train)\n",
    "    print('Random Forest Training score:', model.score(X_train,y_train))\n",
    "    print('Random Forest Test score:    ', model.score(X_test,y_test),'\\n')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predicts the week 44's raw_visit_counts given the trained model and the week 44 known data\n",
    "\n",
    "def presict_week_44(table2, decision_tree_model):\n",
    "    X_predict = table2\n",
    "    poi_id = table2['poi_id']\n",
    "    del X_predict['poi_id']\n",
    "    raw_visit_counts = decision_tree_model.predict(X_predict)\n",
    "\n",
    "\n",
    "    d = pd.DataFrame(columns = ['poi_id', 'raw_visit_counts'])\n",
    "    d['poi_id'] = poi_id\n",
    "    d['raw_visit_counts'] = raw_visit_counts\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    " # Moving Average model with period=3\n",
    "\n",
    "def moving_avg(weekly_pattern):\n",
    "    all_poi = weekly_pattern['poi_id'].unique()\n",
    "    pred_df = pd.DataFrame(all_poi, columns=['poi_id'])\n",
    "    pred_df['week_number'] = 44\n",
    "    pred_df['raw_visit_counts'] = np.nan\n",
    "    pred_df['visits_concentration'] = np.nan\n",
    "    pred_df['distance_from_home'] = np.nan\n",
    "    pred_df['median_dwell'] = np.nan\n",
    "    pred_df['date_start'] = '2020-11-02'\n",
    "    temp=None\n",
    "    temp=weekly_pattern.append(pred_df, ignore_index=True)\n",
    "\n",
    "    for poi in all_poi:\n",
    "        poi_df = temp.loc[temp['poi_id']==poi][['poi_id', \"week_number\", 'raw_visit_counts']].sort_values(by='week_number')\n",
    "        poi_df['MA3']=poi_df['raw_visit_counts'].rolling(4).mean()\n",
    "        if poi_df.shape[0] > 4:\n",
    "            poi_df.iloc[-1, -1]=(poi_df['raw_visit_counts'].iloc[-2]+poi_df['raw_visit_counts'].iloc[-3]+poi_df['raw_visit_counts'].iloc[-4])/3\n",
    "\n",
    "        temp=temp.merge(poi_df, on=['poi_id', 'week_number'], how='left')\n",
    "\n",
    "        temp.drop('raw_visit_counts_y', axis=1, inplace=True)\n",
    "        temp.rename(columns={'raw_visit_counts_x': 'raw_visit_counts'}, inplace=True)\n",
    "\n",
    "        if 'MA3_y' in list(temp.columns):\n",
    "            temp['MA3_x']=np.where(temp['MA3_x'].isnull(), temp['MA3_y'], temp['MA3_x'])\n",
    "            temp.drop('MA3_y', axis=1, inplace=True)\n",
    "            temp.rename(columns={'MA3_x': 'MA3'}, inplace=True)\n",
    "\n",
    "\n",
    "    temp = temp[temp['week_number'] == 44]\n",
    "    result = temp[['poi_id', 'MA3']]\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/google/cloud/bigquery/client.py:441: UserWarning: Cannot create BigQuery Storage client, the dependency google-cloud-bigquery-storage is not installed.\n",
      "  \"Cannot create BigQuery Storage client, the dependency \"\n"
     ]
    }
   ],
   "source": [
    "# Run the functions and store the predicted result in df\n",
    "wp,covid = get_data()\n",
    "combined = add_covid_cases(covid, wp)\n",
    "table = change_format(combined)\n",
    "table2 = prediction_data(combined)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate 532. MiB for an array with shape (1924, 36211) and data type float64",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-113cc7696e28>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#random forest model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrandom_forest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mrand_forest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpresict_week_44\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtable2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-15-17dad4430a59>\u001b[0m in \u001b[0;36mrandom_forest\u001b[0;34m(table)\u001b[0m\n\u001b[1;32m      8\u001b[0m                                                         random_state=0)\n\u001b[1;32m      9\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomForestRegressor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Random Forest Training score:'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Random Forest Test score:    '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/sklearn/ensemble/_forest.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    293\u001b[0m         \"\"\"\n\u001b[1;32m    294\u001b[0m         \u001b[0;31m# Validate or convert input data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"csc\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mDTYPE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m         \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'csc'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mensure_2d\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0msample_weight\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[1;32m    529\u001b[0m                     \u001b[0marray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcasting\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"unsafe\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    530\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 531\u001b[0;31m                     \u001b[0marray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    532\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mComplexWarning\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    533\u001b[0m                 raise ValueError(\"Complex data not supported\\n\"\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/numpy/core/_asarray.py\u001b[0m in \u001b[0;36masarray\u001b[0;34m(a, dtype, order)\u001b[0m\n\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m     \"\"\"\n\u001b[0;32m---> 85\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__array__\u001b[0;34m(self, dtype)\u001b[0m\n\u001b[1;32m   1907\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1908\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__array__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1909\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues_from_object\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1910\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1911\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__array_wrap__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/lib.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.values_from_object\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mvalues\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   5485\u001b[0m         \"\"\"\n\u001b[1;32m   5486\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_consolidate_inplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5487\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_AXIS_REVERSED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5488\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5489\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36mas_array\u001b[0;34m(self, transpose, items)\u001b[0m\n\u001b[1;32m    828\u001b[0m             \u001b[0marr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmgr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mblocks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    829\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 830\u001b[0;31m             \u001b[0marr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmgr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_interleave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    831\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    832\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtranspose\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36m_interleave\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    846\u001b[0m             \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"object\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    847\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 848\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    850\u001b[0m         \u001b[0mitemmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMemoryError\u001b[0m: Unable to allocate 532. MiB for an array with shape (1924, 36211) and data type float64"
     ]
    }
   ],
   "source": [
    "#random forest model\n",
    "model = random_forest(table)\n",
    "rand_forest = presict_week_44(table2, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#moving average model\n",
    "MA3 = moving_avg(wp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'rand_forest' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-9135ddf0db49>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#combine both and store result in df\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrand_forest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmerge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mMA3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mon\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'poi_id'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'poi_id'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'MA3'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'random forest'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'avg'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'MA3'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'random forest'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'MA3'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'rand_forest' is not defined"
     ]
    }
   ],
   "source": [
    "#combine both and store result in df\n",
    "df = rand_forest.merge(MA3, on = 'poi_id')\n",
    "df.columns = ['poi_id','MA3', 'random forest']\n",
    "df['avg'] = (df['MA3'] + df['random forest'])/2\n",
    "del df['MA3']\n",
    "del df['random forest']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results\n",
    "---\n",
    "\n",
    "Storing the result into the csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "## This can also be a good place for you to cleanup any input/output and export your results to a file.\n",
    "\n",
    "df.to_csv('submission_prediction_output.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}