Model_Stacking.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.5"
    },
    "colab": {
      "name": "Model Stacking.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/casperbh96/model-stacking/blob/master/Model_Stacking.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "73zWN1Sb77gf",
        "colab_type": "code",
        "outputId": "c8947b99-8196-4c16-e67a-2101974c1152",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 664
        }
      },
      "source": [
        "!pip install lightgbm xgboost scikit-learn pandas mlxtend --upgrade"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting lightgbm\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0b/9d/ddcb2f43aca194987f1a99e27edf41cf9bc39ea750c3371c2a62698c509a/lightgbm-2.3.1-py2.py3-none-manylinux1_x86_64.whl (1.2MB)\n",
            "\r\u001b[K     |▎                               | 10kB 17.8MB/s eta 0:00:01\r\u001b[K     |▌                               | 20kB 6.6MB/s eta 0:00:01\r\u001b[K     |▉                               | 30kB 8.1MB/s eta 0:00:01\r\u001b[K     |█                               | 40kB 5.8MB/s eta 0:00:01\r\u001b[K     |█▎                              | 51kB 6.3MB/s eta 0:00:01\r\u001b[K     |█▋                              | 61kB 7.3MB/s eta 0:00:01\r\u001b[K     |█▉                              | 71kB 7.8MB/s eta 0:00:01\r\u001b[K     |██                              | 81kB 7.3MB/s eta 0:00:01\r\u001b[K     |██▍                             | 92kB 8.1MB/s eta 0:00:01\r\u001b[K     |██▋                             | 102kB 8.0MB/s eta 0:00:01\r\u001b[K     |███                             | 112kB 8.0MB/s eta 0:00:01\r\u001b[K     |███▏                            | 122kB 8.0MB/s eta 0:00:01\r\u001b[K     |███▍                            | 133kB 8.0MB/s eta 0:00:01\r\u001b[K     |███▊                            | 143kB 8.0MB/s eta 0:00:01\r\u001b[K     |████                            | 153kB 8.0MB/s eta 0:00:01\r\u001b[K     |████▏                           | 163kB 8.0MB/s eta 0:00:01\r\u001b[K     |████▌                           | 174kB 8.0MB/s eta 0:00:01\r\u001b[K     |████▊                           | 184kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████                           | 194kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 204kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 215kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 225kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████                          | 235kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 245kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 256kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 266kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████                         | 276kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 286kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 296kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████                        | 307kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 317kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 327kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 337kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████                       | 348kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 358kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 368kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 378kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████                      | 389kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 399kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 409kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████▉                     | 419kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████                     | 430kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 440kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 450kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████▉                    | 460kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████                    | 471kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 481kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 491kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 501kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 512kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 522kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 532kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 542kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 552kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 563kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 573kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 583kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 593kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████▌                | 604kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 614kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████                | 624kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 634kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 645kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 655kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 665kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 675kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 686kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 696kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 706kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 716kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 727kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 737kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 747kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 757kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 768kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 778kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 788kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 798kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 808kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 819kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 829kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 839kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 849kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 860kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 870kB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 880kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 890kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 901kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 911kB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 921kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 931kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 942kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 952kB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 962kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 972kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 983kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 993kB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 1.0MB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 1.0MB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 1.0MB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▋     | 1.0MB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 1.0MB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▏    | 1.1MB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 1.1MB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 1.1MB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 1.1MB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 1.1MB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 1.1MB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 1.1MB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 1.1MB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 1.1MB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 1.1MB 8.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 1.2MB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 1.2MB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 1.2MB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 1.2MB 8.0MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 1.2MB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 1.2MB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 1.2MB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 1.2MB 8.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 1.2MB 8.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.2MB 8.0MB/s \n",
            "\u001b[?25hRequirement already up-to-date: xgboost in /usr/local/lib/python3.6/dist-packages (0.90)\n",
            "Collecting scikit-learn\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/48/e9fa9e252abcd1447eff6f9257636af31758a6e46fd5ce5d3c879f6907cb/scikit_learn-0.22.1-cp36-cp36m-manylinux1_x86_64.whl (7.0MB)\n",
            "\u001b[K     |████████████████████████████████| 7.1MB 22.0MB/s \n",
            "\u001b[?25hRequirement already up-to-date: pandas in /usr/local/lib/python3.6/dist-packages (0.25.3)\n",
            "Collecting mlxtend\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/04/c362f34f666f0ddc7cf593805e64d64fa670ed96fd9302e68549dd48287d/mlxtend-0.17.0-py2.py3-none-any.whl (1.3MB)\n",
            "\u001b[K     |████████████████████████████████| 1.3MB 47.1MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: scipy in /usr/local/lib/python3.6/dist-packages (from lightgbm) (1.3.3)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from lightgbm) (1.17.4)\n",
            "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn) (0.14.1)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas) (2.6.1)\n",
            "Requirement already satisfied, skipping upgrade: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas) (2018.9)\n",
            "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from mlxtend) (42.0.2)\n",
            "Requirement already satisfied, skipping upgrade: matplotlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from mlxtend) (3.1.2)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.6.1->pandas) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0.0->mlxtend) (2.4.5)\n",
            "Requirement already satisfied, skipping upgrade: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0.0->mlxtend) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0.0->mlxtend) (0.10.0)\n",
            "Installing collected packages: scikit-learn, lightgbm, mlxtend\n",
            "  Found existing installation: scikit-learn 0.21.3\n",
            "    Uninstalling scikit-learn-0.21.3:\n",
            "      Successfully uninstalled scikit-learn-0.21.3\n",
            "  Found existing installation: lightgbm 2.2.3\n",
            "    Uninstalling lightgbm-2.2.3:\n",
            "      Successfully uninstalled lightgbm-2.2.3\n",
            "  Found existing installation: mlxtend 0.14.0\n",
            "    Uninstalling mlxtend-0.14.0:\n",
            "      Successfully uninstalled mlxtend-0.14.0\n",
            "Successfully installed lightgbm-2.3.1 mlxtend-0.17.0 scikit-learn-0.22.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "lightgbm",
                  "mlxtend",
                  "sklearn"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dzx0-ABr75WX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.datasets import load_boston\n",
        "\n",
        "# Suppress warnings for now\n",
        "import warnings\n",
        "warnings.simplefilter(action='ignore', category=UserWarning)\n",
        "warnings.simplefilter(action='ignore', category=FutureWarning)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R5vCI_Fl75Wa",
        "colab_type": "text"
      },
      "source": [
        "# Loading the dataset from Scikit-Learn"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2yw3p1Z675Wb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def dataset_to_df(load):\n",
        "    # Load the input data into the dataframe\n",
        "    df = pd.DataFrame(load.data, columns=load.feature_names)\n",
        "    \n",
        "    # Add the output data into the dataframe\n",
        "    df['label'] = pd.Series(load.target)\n",
        "    \n",
        "    # Return the dataframe\n",
        "    return df"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "OKcGkfdA75Wd",
        "colab_type": "code",
        "outputId": "c8ace8bf-c0c5-404b-b0c0-d9afbba5dee1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "df = dataset_to_df(load_boston())\n",
        "df.head()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CRIM</th>\n",
              "      <th>ZN</th>\n",
              "      <th>INDUS</th>\n",
              "      <th>CHAS</th>\n",
              "      <th>NOX</th>\n",
              "      <th>RM</th>\n",
              "      <th>AGE</th>\n",
              "      <th>DIS</th>\n",
              "      <th>RAD</th>\n",
              "      <th>TAX</th>\n",
              "      <th>PTRATIO</th>\n",
              "      <th>B</th>\n",
              "      <th>LSTAT</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.00632</td>\n",
              "      <td>18.0</td>\n",
              "      <td>2.31</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.538</td>\n",
              "      <td>6.575</td>\n",
              "      <td>65.2</td>\n",
              "      <td>4.0900</td>\n",
              "      <td>1.0</td>\n",
              "      <td>296.0</td>\n",
              "      <td>15.3</td>\n",
              "      <td>396.90</td>\n",
              "      <td>4.98</td>\n",
              "      <td>24.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.02731</td>\n",
              "      <td>0.0</td>\n",
              "      <td>7.07</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.469</td>\n",
              "      <td>6.421</td>\n",
              "      <td>78.9</td>\n",
              "      <td>4.9671</td>\n",
              "      <td>2.0</td>\n",
              "      <td>242.0</td>\n",
              "      <td>17.8</td>\n",
              "      <td>396.90</td>\n",
              "      <td>9.14</td>\n",
              "      <td>21.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.02729</td>\n",
              "      <td>0.0</td>\n",
              "      <td>7.07</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.469</td>\n",
              "      <td>7.185</td>\n",
              "      <td>61.1</td>\n",
              "      <td>4.9671</td>\n",
              "      <td>2.0</td>\n",
              "      <td>242.0</td>\n",
              "      <td>17.8</td>\n",
              "      <td>392.83</td>\n",
              "      <td>4.03</td>\n",
              "      <td>34.7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.03237</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.18</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.458</td>\n",
              "      <td>6.998</td>\n",
              "      <td>45.8</td>\n",
              "      <td>6.0622</td>\n",
              "      <td>3.0</td>\n",
              "      <td>222.0</td>\n",
              "      <td>18.7</td>\n",
              "      <td>394.63</td>\n",
              "      <td>2.94</td>\n",
              "      <td>33.4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.06905</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.18</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.458</td>\n",
              "      <td>7.147</td>\n",
              "      <td>54.2</td>\n",
              "      <td>6.0622</td>\n",
              "      <td>3.0</td>\n",
              "      <td>222.0</td>\n",
              "      <td>18.7</td>\n",
              "      <td>396.90</td>\n",
              "      <td>5.33</td>\n",
              "      <td>36.2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      CRIM    ZN  INDUS  CHAS    NOX  ...    TAX  PTRATIO       B  LSTAT  label\n",
              "0  0.00632  18.0   2.31   0.0  0.538  ...  296.0     15.3  396.90   4.98   24.0\n",
              "1  0.02731   0.0   7.07   0.0  0.469  ...  242.0     17.8  396.90   9.14   21.6\n",
              "2  0.02729   0.0   7.07   0.0  0.469  ...  242.0     17.8  392.83   4.03   34.7\n",
              "3  0.03237   0.0   2.18   0.0  0.458  ...  222.0     18.7  394.63   2.94   33.4\n",
              "4  0.06905   0.0   2.18   0.0  0.458  ...  222.0     18.7  396.90   5.33   36.2\n",
              "\n",
              "[5 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GCIJZVIG75Wg",
        "colab_type": "text"
      },
      "source": [
        "# What does the features mean?\n",
        "\n",
        "The features are described here: https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html\n",
        "\n",
        "1. CRIM - per capita crime rate by town\n",
        "2. ZN - proportion of residential land zoned for lots over 25,000 sq.ft.\n",
        "3. INDUS - proportion of non-retail business acres per town.\n",
        "4. CHAS - Charles River dummy variable (1 if tract bounds river; 0 otherwise)\n",
        "5. NOX - nitric oxides concentration (parts per 10 million)\n",
        "6. RM - average number of rooms per dwelling\n",
        "7. AGE - proportion of owner-occupied units built prior to 1940\n",
        "8. DIS - weighted distances to five Boston employment centres\n",
        "9. RAD - index of accessibility to radial highways\n",
        "10. TAX - full-value property-tax rate per \\$10,000\n",
        "11. PTRATIO - pupil-teacher ratio by town\n",
        "12. B - $1000(Bk - 0.63)^2$ where Bk is the proportion of blacks by town\n",
        "13. LSTAT - % lower status of the population\n",
        "14. (label) MEDV - Median value of owner-occupied homes in $1000's"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BN7de4PZ75Wg",
        "colab_type": "text"
      },
      "source": [
        "# Baseline Predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v565z6M375Wh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Getting the output variable\n",
        "y = df['label']\n",
        "\n",
        "# Getting the input variables\n",
        "X = df.drop(['label'], axis=1)\n",
        "\n",
        "# Diving our input and output into training and testing sets\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "                                    X, y, \n",
        "                                    test_size=0.33, \n",
        "                                    random_state=42\n",
        "                                   )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GcmAVXh275Wj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.metrics import r2_score\n",
        "\n",
        "def algorithm_pipeline(X_train_data, X_test_data, y_train_data, y_test_data, \n",
        "                       model, param_grid, cv=10, scoring_fit='neg_mean_squared_error',\n",
        "                       scoring_test=r2_score, do_probabilities = False):\n",
        "    gs = GridSearchCV(\n",
        "        estimator=model,\n",
        "        param_grid=param_grid, \n",
        "        cv=cv, \n",
        "        n_jobs=-1, \n",
        "        scoring=scoring_fit,\n",
        "        verbose=2\n",
        "    )\n",
        "    fitted_model = gs.fit(X_train_data, y_train_data)\n",
        "    best_model = fitted_model.best_estimator_\n",
        "    \n",
        "    if do_probabilities:\n",
        "      pred = fitted_model.predict_proba(X_test_data)\n",
        "    else:\n",
        "      pred = fitted_model.predict(X_test_data)\n",
        "    \n",
        "    score = scoring_test(y_test_data, pred)\n",
        "    \n",
        "    return [best_model, pred, score]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P--RzLUz75Wl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.ensemble import RandomForestRegressor\n",
        "from lightgbm import LGBMRegressor\n",
        "from xgboost import XGBRegressor\n",
        "\n",
        "# Defining our estimator, the algorithm to optimize\n",
        "models_to_train = [XGBRegressor(), LGBMRegressor(), RandomForestRegressor()]\n",
        "\n",
        "# Defining the hyperparameters to optimize\n",
        "grid_parameters = [\n",
        "    { # XGBoost\n",
        "        'n_estimators': [400, 700, 1000],\n",
        "        'colsample_bytree': [0.7, 0.8],\n",
        "        'max_depth': [15,20,25],\n",
        "        'reg_alpha': [1.1, 1.2, 1.3],\n",
        "        'reg_lambda': [1.1, 1.2, 1.3],\n",
        "        'subsample': [0.7, 0.8, 0.9]\n",
        "    },\n",
        "    { # LightGBM\n",
        "        'n_estimators': [400, 700, 1000],\n",
        "        'learning_rate': [0.12],\n",
        "        'colsample_bytree': [0.7, 0.8],\n",
        "        'max_depth': [4],\n",
        "        'num_leaves': [10, 20],\n",
        "        'reg_alpha': [1.1, 1.2],\n",
        "        'reg_lambda': [1.1, 1.2],\n",
        "        'min_split_gain': [0.3, 0.4],\n",
        "        'subsample': [0.8, 0.9],\n",
        "        'subsample_freq': [10, 20]\n",
        "    }, \n",
        "    { # Random Forest\n",
        "        'max_depth':[3, 5, 10, 13], \n",
        "        'n_estimators':[100, 200, 400, 600, 900],\n",
        "        'max_features':[2, 4, 6, 8, 10]\n",
        "    }\n",
        "]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QEE4dwk175Wn",
        "colab_type": "code",
        "outputId": "1835710e-1db8-4576-d256-cc8fc01a7c2e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        }
      },
      "source": [
        "models_preds_scores = []\n",
        "\n",
        "for i, model in enumerate(models_to_train):\n",
        "    params = grid_parameters[i]\n",
        "    \n",
        "    result = algorithm_pipeline(X_train, X_test, y_train, y_test, \n",
        "                                 model, params, cv=5)\n",
        "    models_preds_scores.append(result)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fitting 5 folds for each of 486 candidates, totalling 2430 fits\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
            "[Parallel(n_jobs=-1)]: Done  37 tasks      | elapsed:    9.1s\n",
            "[Parallel(n_jobs=-1)]: Done 158 tasks      | elapsed:   34.8s\n",
            "[Parallel(n_jobs=-1)]: Done 361 tasks      | elapsed:  1.6min\n",
            "[Parallel(n_jobs=-1)]: Done 644 tasks      | elapsed:  2.8min\n",
            "[Parallel(n_jobs=-1)]: Done 1009 tasks      | elapsed:  4.4min\n",
            "[Parallel(n_jobs=-1)]: Done 1454 tasks      | elapsed:  6.4min\n",
            "[Parallel(n_jobs=-1)]: Done 1981 tasks      | elapsed:  9.0min\n",
            "[Parallel(n_jobs=-1)]: Done 2430 out of 2430 | elapsed: 11.2min finished\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[22:23:46] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "Fitting 5 folds for each of 384 candidates, totalling 1920 fits\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
            "[Parallel(n_jobs=-1)]: Done  70 tasks      | elapsed:    4.5s\n",
            "[Parallel(n_jobs=-1)]: Done 312 tasks      | elapsed:   20.9s\n",
            "[Parallel(n_jobs=-1)]: Done 718 tasks      | elapsed:   49.0s\n",
            "[Parallel(n_jobs=-1)]: Done 1284 tasks      | elapsed:  1.5min\n",
            "[Parallel(n_jobs=-1)]: Done 1920 out of 1920 | elapsed:  2.2min finished\n",
            "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Fitting 5 folds for each of 100 candidates, totalling 500 fits\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[Parallel(n_jobs=-1)]: Done  37 tasks      | elapsed:   15.6s\n",
            "[Parallel(n_jobs=-1)]: Done 158 tasks      | elapsed:  1.4min\n",
            "[Parallel(n_jobs=-1)]: Done 361 tasks      | elapsed:  3.5min\n",
            "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  5.2min finished\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4LBj5Zy1q_Mq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "1ac4f255-7100-46ee-b5c6-8a447866b2df"
      },
      "source": [
        "for result in models_preds_scores:\n",
        "    print('Model: {0}, Score: {1}'.format(type(result[0]).__name__, result[2]))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: XGBRegressor, Score: 0.8954046682953245\n",
            "Model: LGBMRegressor, Score: 0.8678534096374354\n",
            "Model: RandomForestRegressor, Score: 0.8694947742055505\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mJVra5Eg75Wr",
        "colab_type": "text"
      },
      "source": [
        "# Improving baseline with stacking"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HaorxncP75Wt",
        "colab_type": "code",
        "outputId": "f53f5ae9-d621-4118-a965-5a41fb909ecf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        }
      },
      "source": [
        "from mlxtend.regressor import StackingCVRegressor\n",
        "from sklearn.linear_model import Ridge, Lasso\n",
        "from sklearn.svm import SVR\n",
        "\n",
        "xgb = XGBRegressor()\n",
        "lgbm = LGBMRegressor()\n",
        "rf = RandomForestRegressor()\n",
        "ridge = Ridge()\n",
        "lasso = Lasso()\n",
        "svr = SVR(kernel='linear')\n",
        "\n",
        "stack = StackingCVRegressor(regressors=(ridge, lasso, svr, rf, lgbm, xgb),\n",
        "                            meta_regressor=xgb, cv=12,\n",
        "                            use_features_in_secondary=True,\n",
        "                            store_train_meta_features=True,\n",
        "                            shuffle=False,\n",
        "                            random_state=42)\n",
        "\n",
        "stack.fit(X_train, y_train)\n",
        "\n",
        "X_test.columns = ['f0', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12']\n",
        "pred = stack.predict(X_test)\n",
        "score = r2_score(y_test, pred)\n",
        "print(score)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[22:55:40] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "0.9071185724216408\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}