02: H2O_AutoML_Python.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "H2O_AutoML_Python.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gulabpatel/AutoML/blob/main/H2O_AutoML_Python.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ogrpiMMOU9Dw"
      },
      "source": [
        "* H2O AutoML:\n",
        "* Table of contents.\n",
        "* Data support.\n",
        "* Feature engineering.\n",
        "* Model training. Parallel model  testing. Model evaluation and final model creation.\n",
        "* Choosing between AutoML Tables and BigQuery ML.\n",
        "* Model transparency and Cloud Logging.\n",
        "* Explainability. \n",
        "* Test data export.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BiTtGL6IcA8J",
        "outputId": "800d57af-a1c2-4ae9-d76a-6178022b0926",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "#this command might be relevant ony on GPU nodes. But the notebook will run on CPU as well\n",
        "#If running on CPU remove the command wherever it appears\n",
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tue Oct 20 08:53:10 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 455.23.05    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   53C    P8    10W /  70W |      0MiB / 15079MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0GA_-zrlpEa5",
        "outputId": "9fd55b53-adea-409a-db9b-45ffa049a992",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        }
      },
      "source": [
        "#To run H2O AutoML requires java runtime environment\n",
        "!apt-get install default-jre\n",
        "!java -version"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "default-jre is already the newest version (2:1.11-68ubuntu1~18.04.1).\n",
            "default-jre set to manually installed.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 21 not upgraded.\n",
            "openjdk version \"11.0.8\" 2020-07-14\n",
            "OpenJDK Runtime Environment (build 11.0.8+10-post-Ubuntu-0ubuntu118.04.1)\n",
            "OpenJDK 64-Bit Server VM (build 11.0.8+10-post-Ubuntu-0ubuntu118.04.1, mixed mode, sharing)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eHZ8z6iGzZew",
        "outputId": "0d4c66bc-4062-4823-9e22-3a2dc99d215f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 360
        }
      },
      "source": [
        "#Install h2o\n",
        "!pip install h2o"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting h2o\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bb/55/494a42e7509d0874aa444ae4cad0bc7439c936b40e563550fbfabb540275/h2o-3.30.1.3.tar.gz (129.4MB)\n",
            "\u001b[K     |████████████████████████████████| 129.4MB 94kB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from h2o) (2.23.0)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from h2o) (0.8.7)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from h2o) (0.16.0)\n",
            "Collecting colorama>=0.3.8\n",
            "  Downloading https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->h2o) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->h2o) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->h2o) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->h2o) (1.24.3)\n",
            "Building wheels for collected packages: h2o\n",
            "  Building wheel for h2o (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for h2o: filename=h2o-3.30.1.3-py2.py3-none-any.whl size=129446676 sha256=680bde2c255c13af3c7f032e2c8c65a2847ca7790ea6b638251cec60b7d85ea2\n",
            "  Stored in directory: /root/.cache/pip/wheels/3d/e6/07/53ce9be9cb61b33a79cb3ed1fc39f2dae84f6ee6fe1e373e5e\n",
            "Successfully built h2o\n",
            "Installing collected packages: colorama, h2o\n",
            "Successfully installed colorama-0.4.4 h2o-3.30.1.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m0g8oq01zcgM"
      },
      "source": [
        " import h2o"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z2WAtRd4zixl",
        "outputId": "b709e25d-1f1e-4775-c954-a399daee1575",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 545
        }
      },
      "source": [
        "#Initialize h2o\n",
        "h2o.init()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Checking whether there is an H2O instance running at http://localhost:54321 ..... not found.\n",
            "Attempting to start a local H2O server...\n",
            "  Java Version: openjdk version \"11.0.8\" 2020-07-14; OpenJDK Runtime Environment (build 11.0.8+10-post-Ubuntu-0ubuntu118.04.1); OpenJDK 64-Bit Server VM (build 11.0.8+10-post-Ubuntu-0ubuntu118.04.1, mixed mode, sharing)\n",
            "  Starting server from /usr/local/lib/python3.6/dist-packages/h2o/backend/bin/h2o.jar\n",
            "  Ice root: /tmp/tmpxph1cjwj\n",
            "  JVM stdout: /tmp/tmpxph1cjwj/h2o_unknownUser_started_from_python.out\n",
            "  JVM stderr: /tmp/tmpxph1cjwj/h2o_unknownUser_started_from_python.err\n",
            "  Server is running at http://127.0.0.1:54321\n",
            "Connecting to H2O server at http://127.0.0.1:54321 ... successful.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O_cluster_uptime:</td>\n",
              "<td>02 secs</td></tr>\n",
              "<tr><td>H2O_cluster_timezone:</td>\n",
              "<td>Etc/UTC</td></tr>\n",
              "<tr><td>H2O_data_parsing_timezone:</td>\n",
              "<td>UTC</td></tr>\n",
              "<tr><td>H2O_cluster_version:</td>\n",
              "<td>3.30.1.3</td></tr>\n",
              "<tr><td>H2O_cluster_version_age:</td>\n",
              "<td>21 days, 17 hours and 24 minutes </td></tr>\n",
              "<tr><td>H2O_cluster_name:</td>\n",
              "<td>H2O_from_python_unknownUser_cbkpz3</td></tr>\n",
              "<tr><td>H2O_cluster_total_nodes:</td>\n",
              "<td>1</td></tr>\n",
              "<tr><td>H2O_cluster_free_memory:</td>\n",
              "<td>3.180 Gb</td></tr>\n",
              "<tr><td>H2O_cluster_total_cores:</td>\n",
              "<td>2</td></tr>\n",
              "<tr><td>H2O_cluster_allowed_cores:</td>\n",
              "<td>2</td></tr>\n",
              "<tr><td>H2O_cluster_status:</td>\n",
              "<td>accepting new members, healthy</td></tr>\n",
              "<tr><td>H2O_connection_url:</td>\n",
              "<td>http://127.0.0.1:54321</td></tr>\n",
              "<tr><td>H2O_connection_proxy:</td>\n",
              "<td>{\"http\": null, \"https\": null}</td></tr>\n",
              "<tr><td>H2O_internal_security:</td>\n",
              "<td>False</td></tr>\n",
              "<tr><td>H2O_API_Extensions:</td>\n",
              "<td>Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4</td></tr>\n",
              "<tr><td>Python_version:</td>\n",
              "<td>3.6.9 final</td></tr></table></div>"
            ],
            "text/plain": [
              "--------------------------  ------------------------------------------------------------------\n",
              "H2O_cluster_uptime:         02 secs\n",
              "H2O_cluster_timezone:       Etc/UTC\n",
              "H2O_data_parsing_timezone:  UTC\n",
              "H2O_cluster_version:        3.30.1.3\n",
              "H2O_cluster_version_age:    21 days, 17 hours and 24 minutes\n",
              "H2O_cluster_name:           H2O_from_python_unknownUser_cbkpz3\n",
              "H2O_cluster_total_nodes:    1\n",
              "H2O_cluster_free_memory:    3.180 Gb\n",
              "H2O_cluster_total_cores:    2\n",
              "H2O_cluster_allowed_cores:  2\n",
              "H2O_cluster_status:         accepting new members, healthy\n",
              "H2O_connection_url:         http://127.0.0.1:54321\n",
              "H2O_connection_proxy:       {\"http\": null, \"https\": null}\n",
              "H2O_internal_security:      False\n",
              "H2O_API_Extensions:         Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4\n",
              "Python_version:             3.6.9 final\n",
              "--------------------------  ------------------------------------------------------------------"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9kc8CZBJzt6b"
      },
      "source": [
        "from h2o.automl import H2OAutoML"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j-ZCVNqg5DMo"
      },
      "source": [
        "#churn_df = h2o.import_file('https://raw.githubusercontent.com/srivatsan88/YouTubeLI/master/dataset/WA_Fn-UseC_-Telco-Customer-Churn.csv')"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AhpYvYrHMsgQ",
        "outputId": "148925b2-7726-4544-a0f4-3d30a9269443",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "churn_df = h2o.import_file(\"/content/Churn.csv\")"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iig0kW4-M6JQ",
        "outputId": "95eeaf26-410b-414a-9bb8-4792ced17aca",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 440
        }
      },
      "source": [
        "churn_df"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table>\n",
              "<thead>\n",
              "<tr><th>Gender  </th><th>Senior_Citizen  </th><th>Partner  </th><th>Dependents  </th><th style=\"text-align: right;\">  Tenure_Months</th><th>Phone_Service  </th><th>Multiple_Lines  </th><th>Internet_Service  </th><th>Online_Security    </th><th>Online_Backup      </th><th>Device_Protection  </th><th>Tech_Support       </th><th>Streaming_TV       </th><th>Streaming_Movies   </th><th>Contract      </th><th>Paperless_Billing  </th><th>Payment_Method           </th><th style=\"text-align: right;\">  Monthly_Charges</th><th style=\"text-align: right;\">  Total_Charges</th><th>Churn_Label  </th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td>Male    </td><td>No              </td><td>No       </td><td>No          </td><td style=\"text-align: right;\">              2</td><td>Yes            </td><td>No              </td><td>DSL               </td><td>Yes                </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>Yes                </td><td>Mailed check             </td><td style=\"text-align: right;\">            53.85</td><td style=\"text-align: right;\">         108.15</td><td>Yes          </td></tr>\n",
              "<tr><td>Female  </td><td>No              </td><td>No       </td><td>Yes         </td><td style=\"text-align: right;\">              2</td><td>Yes            </td><td>No              </td><td>Fiber optic       </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td style=\"text-align: right;\">            70.7 </td><td style=\"text-align: right;\">         151.65</td><td>Yes          </td></tr>\n",
              "<tr><td>Female  </td><td>No              </td><td>No       </td><td>Yes         </td><td style=\"text-align: right;\">              8</td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td style=\"text-align: right;\">            99.65</td><td style=\"text-align: right;\">         820.5 </td><td>Yes          </td></tr>\n",
              "<tr><td>Female  </td><td>No              </td><td>Yes      </td><td>Yes         </td><td style=\"text-align: right;\">             28</td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>Yes                </td><td>Yes                </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td style=\"text-align: right;\">           104.8 </td><td style=\"text-align: right;\">        3046.05</td><td>Yes          </td></tr>\n",
              "<tr><td>Male    </td><td>No              </td><td>No       </td><td>Yes         </td><td style=\"text-align: right;\">             49</td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>Month-to-month</td><td>Yes                </td><td>Bank transfer (automatic)</td><td style=\"text-align: right;\">           103.7 </td><td style=\"text-align: right;\">        5036.3 </td><td>Yes          </td></tr>\n",
              "<tr><td>Female  </td><td>No              </td><td>Yes      </td><td>No          </td><td style=\"text-align: right;\">             10</td><td>Yes            </td><td>No              </td><td>DSL               </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>No                 </td><td>Credit card (automatic)  </td><td style=\"text-align: right;\">            55.2 </td><td style=\"text-align: right;\">         528.35</td><td>Yes          </td></tr>\n",
              "<tr><td>Male    </td><td>Yes             </td><td>No       </td><td>No          </td><td style=\"text-align: right;\">              1</td><td>No             </td><td>No phone service</td><td>DSL               </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td style=\"text-align: right;\">            39.65</td><td style=\"text-align: right;\">          39.65</td><td>Yes          </td></tr>\n",
              "<tr><td>Male    </td><td>No              </td><td>No       </td><td>No          </td><td style=\"text-align: right;\">              1</td><td>Yes            </td><td>No              </td><td>No                </td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>Month-to-month</td><td>No                 </td><td>Mailed check             </td><td style=\"text-align: right;\">            20.15</td><td style=\"text-align: right;\">          20.15</td><td>Yes          </td></tr>\n",
              "<tr><td>Male    </td><td>No              </td><td>Yes      </td><td>Yes         </td><td style=\"text-align: right;\">             47</td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td style=\"text-align: right;\">            99.35</td><td style=\"text-align: right;\">        4749.15</td><td>Yes          </td></tr>\n",
              "<tr><td>Male    </td><td>No              </td><td>Yes      </td><td>No          </td><td style=\"text-align: right;\">              1</td><td>No             </td><td>No phone service</td><td>DSL               </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>No                 </td><td>Electronic check         </td><td style=\"text-align: right;\">            30.2 </td><td style=\"text-align: right;\">          30.2 </td><td>Yes          </td></tr>\n",
              "</tbody>\n",
              "</table>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aRFEg-iE1LbK",
        "outputId": "9b730a74-902f-4916-81a6-a4fec4bc279c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "churn_df.types"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'Churn_Label': 'enum',\n",
              " 'Contract': 'enum',\n",
              " 'Dependents': 'enum',\n",
              " 'Device_Protection': 'enum',\n",
              " 'Gender': 'enum',\n",
              " 'Internet_Service': 'enum',\n",
              " 'Monthly_Charges': 'real',\n",
              " 'Multiple_Lines': 'enum',\n",
              " 'Online_Backup': 'enum',\n",
              " 'Online_Security': 'enum',\n",
              " 'Paperless_Billing': 'enum',\n",
              " 'Partner': 'enum',\n",
              " 'Payment_Method': 'enum',\n",
              " 'Phone_Service': 'enum',\n",
              " 'Senior_Citizen': 'enum',\n",
              " 'Streaming_Movies': 'enum',\n",
              " 'Streaming_TV': 'enum',\n",
              " 'Tech_Support': 'enum',\n",
              " 'Tenure_Months': 'int',\n",
              " 'Total_Charges': 'real'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R2nAk2Pf5fF1",
        "outputId": "d8d1983b-af43-491f-b849-9df04c6338d8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 655
        }
      },
      "source": [
        "churn_df.describe()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Rows:7043\n",
            "Cols:20\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table>\n",
              "<thead>\n",
              "<tr><th>       </th><th>Gender  </th><th>Senior_Citizen  </th><th>Partner  </th><th>Dependents  </th><th>Tenure_Months     </th><th>Phone_Service  </th><th>Multiple_Lines  </th><th>Internet_Service  </th><th>Online_Security    </th><th>Online_Backup      </th><th>Device_Protection  </th><th>Tech_Support       </th><th>Streaming_TV       </th><th>Streaming_Movies   </th><th>Contract      </th><th>Paperless_Billing  </th><th>Payment_Method           </th><th>Monthly_Charges   </th><th>Total_Charges     </th><th>Churn_Label  </th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td>type   </td><td>enum    </td><td>enum            </td><td>enum     </td><td>enum        </td><td>int               </td><td>enum           </td><td>enum            </td><td>enum              </td><td>enum               </td><td>enum               </td><td>enum               </td><td>enum               </td><td>enum               </td><td>enum               </td><td>enum          </td><td>enum               </td><td>enum                     </td><td>real              </td><td>real              </td><td>enum         </td></tr>\n",
              "<tr><td>mins   </td><td>        </td><td>                </td><td>         </td><td>            </td><td>0.0               </td><td>               </td><td>                </td><td>                  </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>              </td><td>                   </td><td>                         </td><td>18.25             </td><td>18.8              </td><td>             </td></tr>\n",
              "<tr><td>mean   </td><td>        </td><td>                </td><td>         </td><td>            </td><td>32.37114865824223 </td><td>               </td><td>                </td><td>                  </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>              </td><td>                   </td><td>                         </td><td>64.76169246059916 </td><td>2283.3004408418656</td><td>             </td></tr>\n",
              "<tr><td>maxs   </td><td>        </td><td>                </td><td>         </td><td>            </td><td>72.0              </td><td>               </td><td>                </td><td>                  </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>              </td><td>                   </td><td>                         </td><td>118.75            </td><td>8684.8            </td><td>             </td></tr>\n",
              "<tr><td>sigma  </td><td>        </td><td>                </td><td>         </td><td>            </td><td>24.559481023094452</td><td>               </td><td>                </td><td>                  </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>              </td><td>                   </td><td>                         </td><td>30.090047097678482</td><td>2266.771361883145 </td><td>             </td></tr>\n",
              "<tr><td>zeros  </td><td>        </td><td>                </td><td>         </td><td>            </td><td>11                </td><td>               </td><td>                </td><td>                  </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>                   </td><td>              </td><td>                   </td><td>                         </td><td>0                 </td><td>0                 </td><td>             </td></tr>\n",
              "<tr><td>missing</td><td>0       </td><td>0               </td><td>0        </td><td>0           </td><td>0                 </td><td>0              </td><td>0               </td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0                  </td><td>0             </td><td>0                  </td><td>0                        </td><td>0                 </td><td>11                </td><td>0            </td></tr>\n",
              "<tr><td>0      </td><td>Male    </td><td>No              </td><td>No       </td><td>No          </td><td>2.0               </td><td>Yes            </td><td>No              </td><td>DSL               </td><td>Yes                </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>Yes                </td><td>Mailed check             </td><td>53.85             </td><td>108.15            </td><td>Yes          </td></tr>\n",
              "<tr><td>1      </td><td>Female  </td><td>No              </td><td>No       </td><td>Yes         </td><td>2.0               </td><td>Yes            </td><td>No              </td><td>Fiber optic       </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td>70.7              </td><td>151.65            </td><td>Yes          </td></tr>\n",
              "<tr><td>2      </td><td>Female  </td><td>No              </td><td>No       </td><td>Yes         </td><td>8.0               </td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td>99.65             </td><td>820.5             </td><td>Yes          </td></tr>\n",
              "<tr><td>3      </td><td>Female  </td><td>No              </td><td>Yes      </td><td>Yes         </td><td>28.0              </td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>Yes                </td><td>Yes                </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td>104.8             </td><td>3046.05           </td><td>Yes          </td></tr>\n",
              "<tr><td>4      </td><td>Male    </td><td>No              </td><td>No       </td><td>Yes         </td><td>49.0              </td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>Month-to-month</td><td>Yes                </td><td>Bank transfer (automatic)</td><td>103.7             </td><td>5036.3            </td><td>Yes          </td></tr>\n",
              "<tr><td>5      </td><td>Female  </td><td>No              </td><td>Yes      </td><td>No          </td><td>10.0              </td><td>Yes            </td><td>No              </td><td>DSL               </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>No                 </td><td>Credit card (automatic)  </td><td>55.2              </td><td>528.35            </td><td>Yes          </td></tr>\n",
              "<tr><td>6      </td><td>Male    </td><td>Yes             </td><td>No       </td><td>No          </td><td>1.0               </td><td>No             </td><td>No phone service</td><td>DSL               </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td>39.65             </td><td>39.65             </td><td>Yes          </td></tr>\n",
              "<tr><td>7      </td><td>Male    </td><td>No              </td><td>No       </td><td>No          </td><td>1.0               </td><td>Yes            </td><td>No              </td><td>No                </td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>Month-to-month</td><td>No                 </td><td>Mailed check             </td><td>20.15             </td><td>20.15             </td><td>Yes          </td></tr>\n",
              "<tr><td>8      </td><td>Male    </td><td>No              </td><td>Yes      </td><td>Yes         </td><td>47.0              </td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td>99.35             </td><td>4749.15           </td><td>Yes          </td></tr>\n",
              "<tr><td>9      </td><td>Male    </td><td>No              </td><td>Yes      </td><td>No          </td><td>1.0               </td><td>No             </td><td>No phone service</td><td>DSL               </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>No                 </td><td>Electronic check         </td><td>30.2              </td><td>30.2              </td><td>Yes          </td></tr>\n",
              "</tbody>\n",
              "</table>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fzAValVj0Uh0"
      },
      "source": [
        "churn_train,churn_test,churn_valid = churn_df.split_frame(ratios=[.7, .15])"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SHHiJBsU21_U",
        "outputId": "d64acb9c-c755-4d05-a64f-aeb982a49d6b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 440
        }
      },
      "source": [
        "churn_train"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table>\n",
              "<thead>\n",
              "<tr><th>Gender  </th><th>Senior_Citizen  </th><th>Partner  </th><th>Dependents  </th><th style=\"text-align: right;\">  Tenure_Months</th><th>Phone_Service  </th><th>Multiple_Lines  </th><th>Internet_Service  </th><th>Online_Security  </th><th>Online_Backup  </th><th>Device_Protection  </th><th>Tech_Support  </th><th>Streaming_TV  </th><th>Streaming_Movies  </th><th>Contract      </th><th>Paperless_Billing  </th><th>Payment_Method           </th><th style=\"text-align: right;\">  Monthly_Charges</th><th style=\"text-align: right;\">  Total_Charges</th><th>Churn_Label  </th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td>Male    </td><td>No              </td><td>No       </td><td>No          </td><td style=\"text-align: right;\">              2</td><td>Yes            </td><td>No              </td><td>DSL               </td><td>Yes              </td><td>Yes            </td><td>No                 </td><td>No            </td><td>No            </td><td>No                </td><td>Month-to-month</td><td>Yes                </td><td>Mailed check             </td><td style=\"text-align: right;\">            53.85</td><td style=\"text-align: right;\">         108.15</td><td>Yes          </td></tr>\n",
              "<tr><td>Female  </td><td>No              </td><td>No       </td><td>Yes         </td><td style=\"text-align: right;\">              8</td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No               </td><td>No             </td><td>Yes                </td><td>No            </td><td>Yes           </td><td>Yes               </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td style=\"text-align: right;\">            99.65</td><td style=\"text-align: right;\">         820.5 </td><td>Yes          </td></tr>\n",
              "<tr><td>Female  </td><td>No              </td><td>Yes      </td><td>Yes         </td><td style=\"text-align: right;\">             28</td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No               </td><td>No             </td><td>Yes                </td><td>Yes           </td><td>Yes           </td><td>Yes               </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td style=\"text-align: right;\">           104.8 </td><td style=\"text-align: right;\">        3046.05</td><td>Yes          </td></tr>\n",
              "<tr><td>Male    </td><td>No              </td><td>No       </td><td>Yes         </td><td style=\"text-align: right;\">             49</td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No               </td><td>Yes            </td><td>Yes                </td><td>No            </td><td>Yes           </td><td>Yes               </td><td>Month-to-month</td><td>Yes                </td><td>Bank transfer (automatic)</td><td style=\"text-align: right;\">           103.7 </td><td style=\"text-align: right;\">        5036.3 </td><td>Yes          </td></tr>\n",
              "<tr><td>Female  </td><td>No              </td><td>Yes      </td><td>No          </td><td style=\"text-align: right;\">             10</td><td>Yes            </td><td>No              </td><td>DSL               </td><td>No               </td><td>No             </td><td>Yes                </td><td>Yes           </td><td>No            </td><td>No                </td><td>Month-to-month</td><td>No                 </td><td>Credit card (automatic)  </td><td style=\"text-align: right;\">            55.2 </td><td style=\"text-align: right;\">         528.35</td><td>Yes          </td></tr>\n",
              "<tr><td>Male    </td><td>Yes             </td><td>No       </td><td>No          </td><td style=\"text-align: right;\">              1</td><td>No             </td><td>No phone service</td><td>DSL               </td><td>No               </td><td>No             </td><td>Yes                </td><td>No            </td><td>No            </td><td>Yes               </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td style=\"text-align: right;\">            39.65</td><td style=\"text-align: right;\">          39.65</td><td>Yes          </td></tr>\n",
              "<tr><td>Male    </td><td>No              </td><td>Yes      </td><td>Yes         </td><td style=\"text-align: right;\">             47</td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No               </td><td>Yes            </td><td>No                 </td><td>No            </td><td>Yes           </td><td>Yes               </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td style=\"text-align: right;\">            99.35</td><td style=\"text-align: right;\">        4749.15</td><td>Yes          </td></tr>\n",
              "<tr><td>Male    </td><td>No              </td><td>Yes      </td><td>No          </td><td style=\"text-align: right;\">              1</td><td>No             </td><td>No phone service</td><td>DSL               </td><td>No               </td><td>Yes            </td><td>No                 </td><td>No            </td><td>No            </td><td>No                </td><td>Month-to-month</td><td>No                 </td><td>Electronic check         </td><td style=\"text-align: right;\">            30.2 </td><td style=\"text-align: right;\">          30.2 </td><td>Yes          </td></tr>\n",
              "<tr><td>Female  </td><td>No              </td><td>No       </td><td>No          </td><td style=\"text-align: right;\">             17</td><td>Yes            </td><td>No              </td><td>DSL               </td><td>No               </td><td>No             </td><td>No                 </td><td>No            </td><td>Yes           </td><td>Yes               </td><td>Month-to-month</td><td>Yes                </td><td>Mailed check             </td><td style=\"text-align: right;\">            64.7 </td><td style=\"text-align: right;\">        1093.1 </td><td>Yes          </td></tr>\n",
              "<tr><td>Male    </td><td>No              </td><td>No       </td><td>Yes         </td><td style=\"text-align: right;\">             34</td><td>Yes            </td><td>Yes             </td><td>Fiber optic       </td><td>No               </td><td>Yes            </td><td>Yes                </td><td>No            </td><td>Yes           </td><td>Yes               </td><td>Month-to-month</td><td>Yes                </td><td>Electronic check         </td><td style=\"text-align: right;\">           106.35</td><td style=\"text-align: right;\">        3549.25</td><td>Yes          </td></tr>\n",
              "</tbody>\n",
              "</table>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e6QvFazO5tzP"
      },
      "source": [
        "y = \"Churn_Label\"\n",
        "x = churn_df.columns\n",
        "x.remove(y)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ESplmnWT570d"
      },
      "source": [
        "aml = H2OAutoML(max_models = 10, seed = 10, exclude_algos = [\"StackedEnsemble\", \"DeepLearning\"], verbosity=\"info\", nfolds=0)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1uYx3BrXeHtu",
        "outputId": "03c74653-c4a3-4ee5-9525-16837cbeb37b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "#This is just to see whether GPU is being utilized!\n",
        "!nvidia-smi"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tue Oct 20 09:01:23 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 455.23.05    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   38C    P8     9W /  70W |      0MiB / 15079MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fBEls6JqeH0p",
        "outputId": "7c7ed7a1-6b1d-4aea-d311-6eae8358b395",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "aml.train(x = x, y = y, training_frame = churn_train, validation_frame=churn_valid)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "AutoML progress: |\n",
            "09:01:23.697: Project: AutoML_20201020_90123692\n",
            "09:01:23.698: Cross-validation disabled by user: no fold column nor nfolds > 1.\n",
            "09:01:23.699: Setting stopping tolerance adaptively based on the training frame: 0.01432082875315512\n",
            "09:01:23.699: Build control seed: 10\n",
            "09:01:23.699: Since cross-validation is disabled, and no leaderboard frame was provided, automatically split the training data into training and leaderboard frames in the ratio 90/10\n",
            "09:01:23.786: training frame: Frame key: automl_training_py_2_sid_8e46    cols: 20    rows: 4404  chunks: 8    size: 185561  checksum: 1421587068369193328\n",
            "09:01:23.787: validation frame: Frame key: py_4_sid_8e46    cols: 20    rows: 1104  chunks: 8    size: 83263  checksum: 5832880875792482914\n",
            "09:01:23.791: leaderboard frame: Frame key: automl_leaderboard_py_2_sid_8e46    cols: 20    rows: 472  chunks: 8    size: 76549  checksum: -7720541563610321463\n",
            "09:01:23.791: blending frame: NULL\n",
            "09:01:23.792: response column: Churn_Label\n",
            "09:01:23.792: fold column: null\n",
            "09:01:23.792: weights column: null\n",
            "09:01:23.825: Loading execution steps: [{XGBoost : defaults}, {GLM : defaults}, {DRF : [def_1]}, {GBM : defaults}, {DeepLearning : defaults}, {DRF : [XRT]}, {XGBoost : grids}, {GBM : grids}, {DeepLearning : grids}, {GBM : [lr_annealing]}, {XGBoost : [lr_search]}, {StackedEnsemble : defaults}]\n",
            "09:01:23.846: Disabling Algo: DeepLearning as requested by the user.\n",
            "09:01:23.846: Disabling Algo: StackedEnsemble as requested by the user.\n",
            "09:01:23.851: AutoML job created: 2020.10.20 09:01:23.692\n",
            "09:01:23.852: AutoML build started: 2020.10.20 09:01:23.852\n",
            "09:01:23.873: AutoML: starting XGBoost_1_AutoML_20201020_090123 model training\n",
            "\n",
            "█\n",
            "09:01:27.356: New leader: XGBoost_1_AutoML_20201020_090123, auc: 0.8064386640976237\n",
            "09:01:27.356: AutoML: starting XGBoost_2_AutoML_20201020_090123 model training\n",
            "\n",
            "█\n",
            "09:01:29.372: New leader: XGBoost_2_AutoML_20201020_090123, auc: 0.8209010000917515\n",
            "09:01:29.372: AutoML: starting XGBoost_3_AutoML_20201020_090123 model training\n",
            "09:01:30.388: New leader: XGBoost_3_AutoML_20201020_090123, auc: 0.8239288008074135\n",
            "09:01:30.391: AutoML: starting GLM_1_AutoML_20201020_090123 model training\n",
            "\n",
            "██\n",
            "09:01:32.409: New leader: GLM_1_AutoML_20201020_090123, auc: 0.8305693182860814\n",
            "09:01:32.412: AutoML: starting DRF_1_AutoML_20201020_090123 model training\n",
            "\n",
            "██████\n",
            "09:01:37.448: AutoML: starting GBM_1_AutoML_20201020_090123 model training\n",
            "\n",
            "██\n",
            "09:01:39.483: AutoML: starting GBM_2_AutoML_20201020_090123 model training\n",
            "\n",
            "█\n",
            "09:01:41.511: AutoML: starting GBM_3_AutoML_20201020_090123 model training\n",
            "\n",
            "██\n",
            "09:01:43.544: AutoML: starting GBM_4_AutoML_20201020_090123 model training\n",
            "\n",
            "█\n",
            "09:01:45.572: AutoML: starting GBM_5_AutoML_20201020_090123 model training\n",
            "\n",
            "████████████████████████████████████████| 100%\n",
            "\n",
            "09:01:47.594: Skipping StackedEnsemble 'best' due to the exclude_algos option.\n",
            "09:01:47.596: Skipping StackedEnsemble 'all' due to the exclude_algos option.\n",
            "09:01:47.597: Skipping StackedEnsemble 'monotonic' due to the exclude_algos option.\n",
            "09:01:47.597: Actual modeling steps: [{XGBoost : [def_1 (10), def_2 (10), def_3 (10)]}, {GLM : [def_1 (10)]}, {DRF : [def_1 (10)]}, {GBM : [def_1 (10), def_2 (10), def_3 (10), def_4 (10), def_5 (10)]}]\n",
            "09:01:47.597: AutoML build stopped: 2020.10.20 09:01:47.597\n",
            "09:01:47.597: AutoML build done: built 10 models\n",
            "09:01:47.597: AutoML duration: 23.745 sec\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5fHgZtj_eNq9",
        "outputId": "cc239da4-95c7-4f78-d84b-387bd92e48dd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "#Again just to see the utilization of GPU\n",
        "!nvidia-smi"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tue Oct 20 09:01:58 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 455.23.05    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   42C    P0    26W /  70W |    127MiB / 15079MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ALlZEBNd68Nm"
      },
      "source": [
        "lb = aml.leaderboard"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x-bfxNwL6-2l",
        "outputId": "aaad1ab7-654f-4b35-c9d8-010bbb4d6cc1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 270
        }
      },
      "source": [
        "lb.head()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table>\n",
              "<thead>\n",
              "<tr><th>model_id                        </th><th style=\"text-align: right;\">     auc</th><th style=\"text-align: right;\">  logloss</th><th style=\"text-align: right;\">   aucpr</th><th style=\"text-align: right;\">  mean_per_class_error</th><th style=\"text-align: right;\">    rmse</th><th style=\"text-align: right;\">     mse</th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td>GLM_1_AutoML_20201020_090123    </td><td style=\"text-align: right;\">0.830569</td><td style=\"text-align: right;\"> 0.435595</td><td style=\"text-align: right;\">0.610091</td><td style=\"text-align: right;\">              0.233898</td><td style=\"text-align: right;\">0.37634 </td><td style=\"text-align: right;\">0.141632</td></tr>\n",
              "<tr><td>GBM_5_AutoML_20201020_090123    </td><td style=\"text-align: right;\">0.827817</td><td style=\"text-align: right;\"> 0.44155 </td><td style=\"text-align: right;\">0.620153</td><td style=\"text-align: right;\">              0.246169</td><td style=\"text-align: right;\">0.379967</td><td style=\"text-align: right;\">0.144375</td></tr>\n",
              "<tr><td>GBM_2_AutoML_20201020_090123    </td><td style=\"text-align: right;\">0.8255  </td><td style=\"text-align: right;\"> 0.450288</td><td style=\"text-align: right;\">0.621402</td><td style=\"text-align: right;\">              0.253739</td><td style=\"text-align: right;\">0.382054</td><td style=\"text-align: right;\">0.145965</td></tr>\n",
              "<tr><td>DRF_1_AutoML_20201020_090123    </td><td style=\"text-align: right;\">0.824915</td><td style=\"text-align: right;\"> 0.452765</td><td style=\"text-align: right;\">0.607684</td><td style=\"text-align: right;\">              0.226351</td><td style=\"text-align: right;\">0.381202</td><td style=\"text-align: right;\">0.145315</td></tr>\n",
              "<tr><td>XGBoost_3_AutoML_20201020_090123</td><td style=\"text-align: right;\">0.823929</td><td style=\"text-align: right;\"> 0.461706</td><td style=\"text-align: right;\">0.610556</td><td style=\"text-align: right;\">              0.248326</td><td style=\"text-align: right;\">0.385148</td><td style=\"text-align: right;\">0.148339</td></tr>\n",
              "<tr><td>GBM_1_AutoML_20201020_090123    </td><td style=\"text-align: right;\">0.823516</td><td style=\"text-align: right;\"> 0.45059 </td><td style=\"text-align: right;\">0.622331</td><td style=\"text-align: right;\">              0.243623</td><td style=\"text-align: right;\">0.382222</td><td style=\"text-align: right;\">0.146093</td></tr>\n",
              "<tr><td>XGBoost_2_AutoML_20201020_090123</td><td style=\"text-align: right;\">0.820901</td><td style=\"text-align: right;\"> 0.469034</td><td style=\"text-align: right;\">0.611948</td><td style=\"text-align: right;\">              0.257707</td><td style=\"text-align: right;\">0.391398</td><td style=\"text-align: right;\">0.153192</td></tr>\n",
              "<tr><td>GBM_4_AutoML_20201020_090123    </td><td style=\"text-align: right;\">0.820488</td><td style=\"text-align: right;\"> 0.458843</td><td style=\"text-align: right;\">0.632048</td><td style=\"text-align: right;\">              0.239357</td><td style=\"text-align: right;\">0.384851</td><td style=\"text-align: right;\">0.14811 </td></tr>\n",
              "<tr><td>GBM_3_AutoML_20201020_090123    </td><td style=\"text-align: right;\">0.818355</td><td style=\"text-align: right;\"> 0.462416</td><td style=\"text-align: right;\">0.607787</td><td style=\"text-align: right;\">              0.24227 </td><td style=\"text-align: right;\">0.387664</td><td style=\"text-align: right;\">0.150283</td></tr>\n",
              "<tr><td>XGBoost_1_AutoML_20201020_090123</td><td style=\"text-align: right;\">0.806439</td><td style=\"text-align: right;\"> 0.495989</td><td style=\"text-align: right;\">0.59205 </td><td style=\"text-align: right;\">              0.272135</td><td style=\"text-align: right;\">0.400526</td><td style=\"text-align: right;\">0.160421</td></tr>\n",
              "</tbody>\n",
              "</table>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lob8HRvc4bwq",
        "outputId": "8e751f12-6d93-4b40-c388-19c833a69cc4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#Prediction\n",
        "churn_pred=aml.leader.predict(churn_test)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "glm prediction progress: |████████████████████████████████████████████████| 100%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jX50NC-L5NJR",
        "outputId": "2d893f6e-3348-40dc-c154-d70387f334b0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        }
      },
      "source": [
        "churn_pred.head()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table>\n",
              "<thead>\n",
              "<tr><th>predict  </th><th style=\"text-align: right;\">      No</th><th style=\"text-align: right;\">     Yes</th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td>Yes      </td><td style=\"text-align: right;\">0.617169</td><td style=\"text-align: right;\">0.382831</td></tr>\n",
              "<tr><td>No       </td><td style=\"text-align: right;\">0.780593</td><td style=\"text-align: right;\">0.219407</td></tr>\n",
              "<tr><td>No       </td><td style=\"text-align: right;\">0.80075 </td><td style=\"text-align: right;\">0.19925 </td></tr>\n",
              "<tr><td>No       </td><td style=\"text-align: right;\">0.744904</td><td style=\"text-align: right;\">0.255096</td></tr>\n",
              "<tr><td>No       </td><td style=\"text-align: right;\">0.898151</td><td style=\"text-align: right;\">0.101849</td></tr>\n",
              "<tr><td>Yes      </td><td style=\"text-align: right;\">0.300466</td><td style=\"text-align: right;\">0.699534</td></tr>\n",
              "<tr><td>Yes      </td><td style=\"text-align: right;\">0.563123</td><td style=\"text-align: right;\">0.436877</td></tr>\n",
              "<tr><td>No       </td><td style=\"text-align: right;\">0.723504</td><td style=\"text-align: right;\">0.276496</td></tr>\n",
              "<tr><td>Yes      </td><td style=\"text-align: right;\">0.322257</td><td style=\"text-align: right;\">0.677743</td></tr>\n",
              "<tr><td>Yes      </td><td style=\"text-align: right;\">0.62853 </td><td style=\"text-align: right;\">0.37147 </td></tr>\n",
              "</tbody>\n",
              "</table>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GVTyHEfe6whc",
        "outputId": "46d1442a-348c-40e0-ee90-93ddfada6fb9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "aml.leader.model_performance(churn_test)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "ModelMetricsBinomialGLM: glm\n",
            "** Reported on test data. **\n",
            "\n",
            "MSE: 0.135339227274861\n",
            "RMSE: 0.3678848016361385\n",
            "LogLoss: 0.41816394388048317\n",
            "Null degrees of freedom: 1062\n",
            "Residual degrees of freedom: 1016\n",
            "Null deviance: 1213.581423519483\n",
            "Residual deviance: 889.016544689907\n",
            "AIC: 983.016544689907\n",
            "AUC: 0.8393489865208663\n",
            "AUCPR: 0.6458452583126004\n",
            "Gini: 0.6786979730417326\n",
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.2466313986481698: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>578.0</td>\n",
              "      <td>211.0</td>\n",
              "      <td>0.2674</td>\n",
              "      <td>(211.0/789.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>54.0</td>\n",
              "      <td>220.0</td>\n",
              "      <td>0.1971</td>\n",
              "      <td>(54.0/274.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>632.0</td>\n",
              "      <td>431.0</td>\n",
              "      <td>0.2493</td>\n",
              "      <td>(265.0/1063.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             No    Yes   Error             Rate\n",
              "0     No  578.0  211.0  0.2674    (211.0/789.0)\n",
              "1    Yes   54.0  220.0  0.1971     (54.0/274.0)\n",
              "2  Total  632.0  431.0  0.2493   (265.0/1063.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Maximum Metrics: Maximum metrics at their respective thresholds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>threshold</th>\n",
              "      <th>value</th>\n",
              "      <th>idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>max f1</td>\n",
              "      <td>0.246631</td>\n",
              "      <td>0.624113</td>\n",
              "      <td>234.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>max f2</td>\n",
              "      <td>0.145116</td>\n",
              "      <td>0.738153</td>\n",
              "      <td>283.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>max f0point5</td>\n",
              "      <td>0.520835</td>\n",
              "      <td>0.639746</td>\n",
              "      <td>123.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>max accuracy</td>\n",
              "      <td>0.520835</td>\n",
              "      <td>0.812794</td>\n",
              "      <td>123.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>max precision</td>\n",
              "      <td>0.901031</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>max recall</td>\n",
              "      <td>0.007878</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>391.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>max specificity</td>\n",
              "      <td>0.901031</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>max absolute_mcc</td>\n",
              "      <td>0.498793</td>\n",
              "      <td>0.478023</td>\n",
              "      <td>133.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>max min_per_class_accuracy</td>\n",
              "      <td>0.280751</td>\n",
              "      <td>0.759189</td>\n",
              "      <td>218.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>max mean_per_class_accuracy</td>\n",
              "      <td>0.246631</td>\n",
              "      <td>0.767746</td>\n",
              "      <td>234.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>max tns</td>\n",
              "      <td>0.901031</td>\n",
              "      <td>789.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>max fns</td>\n",
              "      <td>0.901031</td>\n",
              "      <td>273.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>max fps</td>\n",
              "      <td>0.001987</td>\n",
              "      <td>789.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>max tps</td>\n",
              "      <td>0.007878</td>\n",
              "      <td>274.000000</td>\n",
              "      <td>391.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>max tnr</td>\n",
              "      <td>0.901031</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>max fnr</td>\n",
              "      <td>0.901031</td>\n",
              "      <td>0.996350</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>max fpr</td>\n",
              "      <td>0.001987</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>max tpr</td>\n",
              "      <td>0.007878</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>391.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         metric  threshold       value    idx\n",
              "0                        max f1   0.246631    0.624113  234.0\n",
              "1                        max f2   0.145116    0.738153  283.0\n",
              "2                  max f0point5   0.520835    0.639746  123.0\n",
              "3                  max accuracy   0.520835    0.812794  123.0\n",
              "4                 max precision   0.901031    1.000000    0.0\n",
              "5                    max recall   0.007878    1.000000  391.0\n",
              "6               max specificity   0.901031    1.000000    0.0\n",
              "7              max absolute_mcc   0.498793    0.478023  133.0\n",
              "8    max min_per_class_accuracy   0.280751    0.759189  218.0\n",
              "9   max mean_per_class_accuracy   0.246631    0.767746  234.0\n",
              "10                      max tns   0.901031  789.000000    0.0\n",
              "11                      max fns   0.901031  273.000000    0.0\n",
              "12                      max fps   0.001987  789.000000  399.0\n",
              "13                      max tps   0.007878  274.000000  391.0\n",
              "14                      max tnr   0.901031    1.000000    0.0\n",
              "15                      max fnr   0.901031    0.996350    0.0\n",
              "16                      max fpr   0.001987    1.000000  399.0\n",
              "17                      max tpr   0.007878    1.000000  391.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Gains/Lift Table: Avg response rate: 25.78 %, avg score: 25.73 %\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>group</th>\n",
              "      <th>cumulative_data_fraction</th>\n",
              "      <th>lower_threshold</th>\n",
              "      <th>lift</th>\n",
              "      <th>cumulative_lift</th>\n",
              "      <th>response_rate</th>\n",
              "      <th>score</th>\n",
              "      <th>cumulative_response_rate</th>\n",
              "      <th>cumulative_score</th>\n",
              "      <th>capture_rate</th>\n",
              "      <th>cumulative_capture_rate</th>\n",
              "      <th>gain</th>\n",
              "      <th>cumulative_gain</th>\n",
              "      <th>kolmogorov_smirnov</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0.010348</td>\n",
              "      <td>0.829778</td>\n",
              "      <td>3.174187</td>\n",
              "      <td>3.174187</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.858948</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.858948</td>\n",
              "      <td>0.032847</td>\n",
              "      <td>0.032847</td>\n",
              "      <td>217.418713</td>\n",
              "      <td>217.418713</td>\n",
              "      <td>0.030312</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0.020696</td>\n",
              "      <td>0.806996</td>\n",
              "      <td>3.174187</td>\n",
              "      <td>3.174187</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.816365</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.837656</td>\n",
              "      <td>0.032847</td>\n",
              "      <td>0.065693</td>\n",
              "      <td>217.418713</td>\n",
              "      <td>217.418713</td>\n",
              "      <td>0.060624</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>0.030103</td>\n",
              "      <td>0.782344</td>\n",
              "      <td>3.879562</td>\n",
              "      <td>3.394617</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.794991</td>\n",
              "      <td>0.875000</td>\n",
              "      <td>0.824323</td>\n",
              "      <td>0.036496</td>\n",
              "      <td>0.102190</td>\n",
              "      <td>287.956204</td>\n",
              "      <td>239.461679</td>\n",
              "      <td>0.097120</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>0.040452</td>\n",
              "      <td>0.762494</td>\n",
              "      <td>2.821500</td>\n",
              "      <td>3.248005</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.770739</td>\n",
              "      <td>0.837209</td>\n",
              "      <td>0.810616</td>\n",
              "      <td>0.029197</td>\n",
              "      <td>0.131387</td>\n",
              "      <td>182.149967</td>\n",
              "      <td>224.800543</td>\n",
              "      <td>0.122515</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>0.050800</td>\n",
              "      <td>0.747306</td>\n",
              "      <td>3.174187</td>\n",
              "      <td>3.232968</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.754036</td>\n",
              "      <td>0.833333</td>\n",
              "      <td>0.799090</td>\n",
              "      <td>0.032847</td>\n",
              "      <td>0.164234</td>\n",
              "      <td>217.418713</td>\n",
              "      <td>223.296837</td>\n",
              "      <td>0.152827</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>6</td>\n",
              "      <td>0.100659</td>\n",
              "      <td>0.677191</td>\n",
              "      <td>2.488776</td>\n",
              "      <td>2.864350</td>\n",
              "      <td>0.641509</td>\n",
              "      <td>0.713263</td>\n",
              "      <td>0.738318</td>\n",
              "      <td>0.756578</td>\n",
              "      <td>0.124088</td>\n",
              "      <td>0.288321</td>\n",
              "      <td>148.877565</td>\n",
              "      <td>186.434955</td>\n",
              "      <td>0.252833</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>7</td>\n",
              "      <td>0.150517</td>\n",
              "      <td>0.600458</td>\n",
              "      <td>2.415576</td>\n",
              "      <td>2.715693</td>\n",
              "      <td>0.622642</td>\n",
              "      <td>0.638739</td>\n",
              "      <td>0.700000</td>\n",
              "      <td>0.717544</td>\n",
              "      <td>0.120438</td>\n",
              "      <td>0.408759</td>\n",
              "      <td>141.557637</td>\n",
              "      <td>171.569343</td>\n",
              "      <td>0.347923</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>8</td>\n",
              "      <td>0.200376</td>\n",
              "      <td>0.512637</td>\n",
              "      <td>2.269178</td>\n",
              "      <td>2.604589</td>\n",
              "      <td>0.584906</td>\n",
              "      <td>0.557052</td>\n",
              "      <td>0.671362</td>\n",
              "      <td>0.677609</td>\n",
              "      <td>0.113139</td>\n",
              "      <td>0.521898</td>\n",
              "      <td>126.917780</td>\n",
              "      <td>160.458860</td>\n",
              "      <td>0.433178</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>9</td>\n",
              "      <td>0.300094</td>\n",
              "      <td>0.367989</td>\n",
              "      <td>1.463986</td>\n",
              "      <td>2.225579</td>\n",
              "      <td>0.377358</td>\n",
              "      <td>0.436850</td>\n",
              "      <td>0.573668</td>\n",
              "      <td>0.597608</td>\n",
              "      <td>0.145985</td>\n",
              "      <td>0.667883</td>\n",
              "      <td>46.398568</td>\n",
              "      <td>122.557948</td>\n",
              "      <td>0.495513</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>10</td>\n",
              "      <td>0.399812</td>\n",
              "      <td>0.250307</td>\n",
              "      <td>1.244388</td>\n",
              "      <td>1.980859</td>\n",
              "      <td>0.320755</td>\n",
              "      <td>0.307436</td>\n",
              "      <td>0.510588</td>\n",
              "      <td>0.525235</td>\n",
              "      <td>0.124088</td>\n",
              "      <td>0.791971</td>\n",
              "      <td>24.438783</td>\n",
              "      <td>98.085874</td>\n",
              "      <td>0.528346</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>11</td>\n",
              "      <td>0.500470</td>\n",
              "      <td>0.157186</td>\n",
              "      <td>0.761409</td>\n",
              "      <td>1.735594</td>\n",
              "      <td>0.196262</td>\n",
              "      <td>0.203419</td>\n",
              "      <td>0.447368</td>\n",
              "      <td>0.460509</td>\n",
              "      <td>0.076642</td>\n",
              "      <td>0.868613</td>\n",
              "      <td>-23.859063</td>\n",
              "      <td>73.559355</td>\n",
              "      <td>0.495990</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>12</td>\n",
              "      <td>0.600188</td>\n",
              "      <td>0.099092</td>\n",
              "      <td>0.512395</td>\n",
              "      <td>1.532366</td>\n",
              "      <td>0.132075</td>\n",
              "      <td>0.125569</td>\n",
              "      <td>0.394984</td>\n",
              "      <td>0.404861</td>\n",
              "      <td>0.051095</td>\n",
              "      <td>0.919708</td>\n",
              "      <td>-48.760501</td>\n",
              "      <td>53.236620</td>\n",
              "      <td>0.430481</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>13</td>\n",
              "      <td>0.699906</td>\n",
              "      <td>0.056625</td>\n",
              "      <td>0.439196</td>\n",
              "      <td>1.376619</td>\n",
              "      <td>0.113208</td>\n",
              "      <td>0.076581</td>\n",
              "      <td>0.354839</td>\n",
              "      <td>0.358090</td>\n",
              "      <td>0.043796</td>\n",
              "      <td>0.963504</td>\n",
              "      <td>-56.080430</td>\n",
              "      <td>37.661879</td>\n",
              "      <td>0.355139</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>14</td>\n",
              "      <td>0.799624</td>\n",
              "      <td>0.026769</td>\n",
              "      <td>0.256197</td>\n",
              "      <td>1.236896</td>\n",
              "      <td>0.066038</td>\n",
              "      <td>0.040860</td>\n",
              "      <td>0.318824</td>\n",
              "      <td>0.318529</td>\n",
              "      <td>0.025547</td>\n",
              "      <td>0.989051</td>\n",
              "      <td>-74.380251</td>\n",
              "      <td>23.689566</td>\n",
              "      <td>0.255211</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>15</td>\n",
              "      <td>0.899341</td>\n",
              "      <td>0.012239</td>\n",
              "      <td>0.073199</td>\n",
              "      <td>1.107867</td>\n",
              "      <td>0.018868</td>\n",
              "      <td>0.019056</td>\n",
              "      <td>0.285565</td>\n",
              "      <td>0.285324</td>\n",
              "      <td>0.007299</td>\n",
              "      <td>0.996350</td>\n",
              "      <td>-92.680072</td>\n",
              "      <td>10.786657</td>\n",
              "      <td>0.130698</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>16</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.001696</td>\n",
              "      <td>0.036258</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.009346</td>\n",
              "      <td>0.006548</td>\n",
              "      <td>0.257761</td>\n",
              "      <td>0.257263</td>\n",
              "      <td>0.003650</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-96.374241</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    group  cumulative_data_fraction  ...  cumulative_gain  kolmogorov_smirnov\n",
              "0       1                  0.010348  ...       217.418713            0.030312\n",
              "1       2                  0.020696  ...       217.418713            0.060624\n",
              "2       3                  0.030103  ...       239.461679            0.097120\n",
              "3       4                  0.040452  ...       224.800543            0.122515\n",
              "4       5                  0.050800  ...       223.296837            0.152827\n",
              "5       6                  0.100659  ...       186.434955            0.252833\n",
              "6       7                  0.150517  ...       171.569343            0.347923\n",
              "7       8                  0.200376  ...       160.458860            0.433178\n",
              "8       9                  0.300094  ...       122.557948            0.495513\n",
              "9      10                  0.399812  ...        98.085874            0.528346\n",
              "10     11                  0.500470  ...        73.559355            0.495990\n",
              "11     12                  0.600188  ...        53.236620            0.430481\n",
              "12     13                  0.699906  ...        37.661879            0.355139\n",
              "13     14                  0.799624  ...        23.689566            0.255211\n",
              "14     15                  0.899341  ...        10.786657            0.130698\n",
              "15     16                  1.000000  ...         0.000000            0.000000\n",
              "\n",
              "[16 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kl3m9BrA8whW"
      },
      "source": [
        "#Take all the model id to analysis individually\n",
        "model_ids = list(aml.leaderboard['model_id'].as_data_frame().iloc[:,0])\n",
        "#se = h2o.get_model([mid for mid in model_ids if \"StackedEnsemble_AllModels\" in mid][0])\n",
        "#metalearner = h2o.get_model(se.metalearner()['name'])"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S_7RYNenT527",
        "outputId": "bcb80a21-dd32-4fce-f911-f06c4dcc3bdc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "model_ids"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['GLM_1_AutoML_20201020_090123',\n",
              " 'GBM_5_AutoML_20201020_090123',\n",
              " 'GBM_2_AutoML_20201020_090123',\n",
              " 'DRF_1_AutoML_20201020_090123',\n",
              " 'XGBoost_3_AutoML_20201020_090123',\n",
              " 'GBM_1_AutoML_20201020_090123',\n",
              " 'XGBoost_2_AutoML_20201020_090123',\n",
              " 'GBM_4_AutoML_20201020_090123',\n",
              " 'GBM_3_AutoML_20201020_090123',\n",
              " 'XGBoost_1_AutoML_20201020_090123']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vOxeivTyUCeq",
        "outputId": "072652ea-fa4a-4c53-e90b-93765b4ea53a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "h2o.get_model([mid for mid in model_ids if \"XGBoost\" in mid][0])"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model Details\n",
            "=============\n",
            "H2OXGBoostEstimator :  XGBoost\n",
            "Model Key:  XGBoost_3_AutoML_20201020_090123\n",
            "\n",
            "\n",
            "Model Summary: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>number_of_trees</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>35.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     number_of_trees\n",
              "0               35.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "ModelMetricsBinomial: xgboost\n",
            "** Reported on train data. **\n",
            "\n",
            "MSE: 0.09399571002559544\n",
            "RMSE: 0.30658719807845114\n",
            "LogLoss: 0.29949752998721063\n",
            "Mean Per-Class Error: 0.1452000631438206\n",
            "AUC: 0.9305935784005755\n",
            "AUCPR: 0.8299469678919649\n",
            "Gini: 0.8611871568011511\n",
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.3705852244581495: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>2841.0</td>\n",
              "      <td>397.0</td>\n",
              "      <td>0.1226</td>\n",
              "      <td>(397.0/3238.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>201.0</td>\n",
              "      <td>965.0</td>\n",
              "      <td>0.1724</td>\n",
              "      <td>(201.0/1166.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>3042.0</td>\n",
              "      <td>1362.0</td>\n",
              "      <td>0.1358</td>\n",
              "      <td>(598.0/4404.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              No     Yes   Error             Rate\n",
              "0     No  2841.0   397.0  0.1226   (397.0/3238.0)\n",
              "1    Yes   201.0   965.0  0.1724   (201.0/1166.0)\n",
              "2  Total  3042.0  1362.0  0.1358   (598.0/4404.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Maximum Metrics: Maximum metrics at their respective thresholds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>threshold</th>\n",
              "      <th>value</th>\n",
              "      <th>idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>max f1</td>\n",
              "      <td>0.370585</td>\n",
              "      <td>0.763449</td>\n",
              "      <td>208.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>max f2</td>\n",
              "      <td>0.197192</td>\n",
              "      <td>0.829819</td>\n",
              "      <td>279.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>max f0point5</td>\n",
              "      <td>0.572713</td>\n",
              "      <td>0.774095</td>\n",
              "      <td>133.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>max accuracy</td>\n",
              "      <td>0.456477</td>\n",
              "      <td>0.870345</td>\n",
              "      <td>176.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>max precision</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>max recall</td>\n",
              "      <td>0.017452</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>379.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>max specificity</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>max absolute_mcc</td>\n",
              "      <td>0.370585</td>\n",
              "      <td>0.672999</td>\n",
              "      <td>208.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>max min_per_class_accuracy</td>\n",
              "      <td>0.337709</td>\n",
              "      <td>0.854540</td>\n",
              "      <td>222.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>max mean_per_class_accuracy</td>\n",
              "      <td>0.337709</td>\n",
              "      <td>0.854800</td>\n",
              "      <td>222.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>max tns</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>3238.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>max fns</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>1165.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>max fps</td>\n",
              "      <td>0.001026</td>\n",
              "      <td>3238.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>max tps</td>\n",
              "      <td>0.017452</td>\n",
              "      <td>1166.000000</td>\n",
              "      <td>379.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>max tnr</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>max fnr</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>0.999142</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>max fpr</td>\n",
              "      <td>0.001026</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>max tpr</td>\n",
              "      <td>0.017452</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>379.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         metric  threshold        value    idx\n",
              "0                        max f1   0.370585     0.763449  208.0\n",
              "1                        max f2   0.197192     0.829819  279.0\n",
              "2                  max f0point5   0.572713     0.774095  133.0\n",
              "3                  max accuracy   0.456477     0.870345  176.0\n",
              "4                 max precision   0.986969     1.000000    0.0\n",
              "5                    max recall   0.017452     1.000000  379.0\n",
              "6               max specificity   0.986969     1.000000    0.0\n",
              "7              max absolute_mcc   0.370585     0.672999  208.0\n",
              "8    max min_per_class_accuracy   0.337709     0.854540  222.0\n",
              "9   max mean_per_class_accuracy   0.337709     0.854800  222.0\n",
              "10                      max tns   0.986969  3238.000000    0.0\n",
              "11                      max fns   0.986969  1165.000000    0.0\n",
              "12                      max fps   0.001026  3238.000000  399.0\n",
              "13                      max tps   0.017452  1166.000000  379.0\n",
              "14                      max tnr   0.986969     1.000000    0.0\n",
              "15                      max fnr   0.986969     0.999142    0.0\n",
              "16                      max fpr   0.001026     1.000000  399.0\n",
              "17                      max tpr   0.017452     1.000000  379.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Gains/Lift Table: Avg response rate: 26.48 %, avg score: 26.59 %\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>group</th>\n",
              "      <th>cumulative_data_fraction</th>\n",
              "      <th>lower_threshold</th>\n",
              "      <th>lift</th>\n",
              "      <th>cumulative_lift</th>\n",
              "      <th>response_rate</th>\n",
              "      <th>score</th>\n",
              "      <th>cumulative_response_rate</th>\n",
              "      <th>cumulative_score</th>\n",
              "      <th>capture_rate</th>\n",
              "      <th>cumulative_capture_rate</th>\n",
              "      <th>gain</th>\n",
              "      <th>cumulative_gain</th>\n",
              "      <th>kolmogorov_smirnov</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0.010218</td>\n",
              "      <td>0.931747</td>\n",
              "      <td>3.777015</td>\n",
              "      <td>3.777015</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.956726</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.956726</td>\n",
              "      <td>0.038593</td>\n",
              "      <td>0.038593</td>\n",
              "      <td>277.701544</td>\n",
              "      <td>277.701544</td>\n",
              "      <td>0.038593</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0.020209</td>\n",
              "      <td>0.907316</td>\n",
              "      <td>3.605333</td>\n",
              "      <td>3.692139</td>\n",
              "      <td>0.954545</td>\n",
              "      <td>0.918276</td>\n",
              "      <td>0.977528</td>\n",
              "      <td>0.937717</td>\n",
              "      <td>0.036021</td>\n",
              "      <td>0.074614</td>\n",
              "      <td>260.533292</td>\n",
              "      <td>269.213869</td>\n",
              "      <td>0.073996</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>0.030200</td>\n",
              "      <td>0.889492</td>\n",
              "      <td>3.691174</td>\n",
              "      <td>3.691820</td>\n",
              "      <td>0.977273</td>\n",
              "      <td>0.899013</td>\n",
              "      <td>0.977444</td>\n",
              "      <td>0.924913</td>\n",
              "      <td>0.036878</td>\n",
              "      <td>0.111492</td>\n",
              "      <td>269.117418</td>\n",
              "      <td>269.181960</td>\n",
              "      <td>0.110566</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>0.040191</td>\n",
              "      <td>0.860961</td>\n",
              "      <td>3.519492</td>\n",
              "      <td>3.648981</td>\n",
              "      <td>0.931818</td>\n",
              "      <td>0.875097</td>\n",
              "      <td>0.966102</td>\n",
              "      <td>0.912529</td>\n",
              "      <td>0.035163</td>\n",
              "      <td>0.146655</td>\n",
              "      <td>251.949166</td>\n",
              "      <td>264.898102</td>\n",
              "      <td>0.144802</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>0.050182</td>\n",
              "      <td>0.840925</td>\n",
              "      <td>3.691174</td>\n",
              "      <td>3.657381</td>\n",
              "      <td>0.977273</td>\n",
              "      <td>0.850945</td>\n",
              "      <td>0.968326</td>\n",
              "      <td>0.900268</td>\n",
              "      <td>0.036878</td>\n",
              "      <td>0.183533</td>\n",
              "      <td>269.117418</td>\n",
              "      <td>265.738146</td>\n",
              "      <td>0.181372</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>6</td>\n",
              "      <td>0.100136</td>\n",
              "      <td>0.745977</td>\n",
              "      <td>3.279136</td>\n",
              "      <td>3.468688</td>\n",
              "      <td>0.868182</td>\n",
              "      <td>0.792359</td>\n",
              "      <td>0.918367</td>\n",
              "      <td>0.846436</td>\n",
              "      <td>0.163808</td>\n",
              "      <td>0.347341</td>\n",
              "      <td>227.913613</td>\n",
              "      <td>246.868765</td>\n",
              "      <td>0.336223</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>7</td>\n",
              "      <td>0.150091</td>\n",
              "      <td>0.647722</td>\n",
              "      <td>2.970108</td>\n",
              "      <td>3.302746</td>\n",
              "      <td>0.786364</td>\n",
              "      <td>0.695603</td>\n",
              "      <td>0.874433</td>\n",
              "      <td>0.796234</td>\n",
              "      <td>0.148370</td>\n",
              "      <td>0.495712</td>\n",
              "      <td>197.010759</td>\n",
              "      <td>230.274572</td>\n",
              "      <td>0.470079</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>8</td>\n",
              "      <td>0.200045</td>\n",
              "      <td>0.558343</td>\n",
              "      <td>2.523733</td>\n",
              "      <td>3.108214</td>\n",
              "      <td>0.668182</td>\n",
              "      <td>0.605589</td>\n",
              "      <td>0.822928</td>\n",
              "      <td>0.748627</td>\n",
              "      <td>0.126072</td>\n",
              "      <td>0.621784</td>\n",
              "      <td>152.373304</td>\n",
              "      <td>210.821361</td>\n",
              "      <td>0.573606</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>9</td>\n",
              "      <td>0.299955</td>\n",
              "      <td>0.380472</td>\n",
              "      <td>1.914260</td>\n",
              "      <td>2.710530</td>\n",
              "      <td>0.506818</td>\n",
              "      <td>0.466967</td>\n",
              "      <td>0.717638</td>\n",
              "      <td>0.654811</td>\n",
              "      <td>0.191252</td>\n",
              "      <td>0.813036</td>\n",
              "      <td>91.426010</td>\n",
              "      <td>171.053038</td>\n",
              "      <td>0.697841</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>10</td>\n",
              "      <td>0.400091</td>\n",
              "      <td>0.253055</td>\n",
              "      <td>0.959242</td>\n",
              "      <td>2.272211</td>\n",
              "      <td>0.253968</td>\n",
              "      <td>0.315455</td>\n",
              "      <td>0.601589</td>\n",
              "      <td>0.569876</td>\n",
              "      <td>0.096055</td>\n",
              "      <td>0.909091</td>\n",
              "      <td>-4.075798</td>\n",
              "      <td>127.221133</td>\n",
              "      <td>0.692290</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>11</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.144239</td>\n",
              "      <td>0.583721</td>\n",
              "      <td>1.934820</td>\n",
              "      <td>0.154545</td>\n",
              "      <td>0.196314</td>\n",
              "      <td>0.512262</td>\n",
              "      <td>0.495232</td>\n",
              "      <td>0.058319</td>\n",
              "      <td>0.967410</td>\n",
              "      <td>-41.627943</td>\n",
              "      <td>93.481990</td>\n",
              "      <td>0.635724</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>12</td>\n",
              "      <td>0.599909</td>\n",
              "      <td>0.070011</td>\n",
              "      <td>0.214603</td>\n",
              "      <td>1.648334</td>\n",
              "      <td>0.056818</td>\n",
              "      <td>0.104651</td>\n",
              "      <td>0.436412</td>\n",
              "      <td>0.430184</td>\n",
              "      <td>0.021441</td>\n",
              "      <td>0.988851</td>\n",
              "      <td>-78.539685</td>\n",
              "      <td>64.833414</td>\n",
              "      <td>0.528999</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>13</td>\n",
              "      <td>0.700045</td>\n",
              "      <td>0.030656</td>\n",
              "      <td>0.077082</td>\n",
              "      <td>1.423578</td>\n",
              "      <td>0.020408</td>\n",
              "      <td>0.047875</td>\n",
              "      <td>0.376906</td>\n",
              "      <td>0.375498</td>\n",
              "      <td>0.007719</td>\n",
              "      <td>0.996569</td>\n",
              "      <td>-92.291805</td>\n",
              "      <td>42.357831</td>\n",
              "      <td>0.403302</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>14</td>\n",
              "      <td>0.799955</td>\n",
              "      <td>0.012430</td>\n",
              "      <td>0.034337</td>\n",
              "      <td>1.250071</td>\n",
              "      <td>0.009091</td>\n",
              "      <td>0.020357</td>\n",
              "      <td>0.330968</td>\n",
              "      <td>0.331143</td>\n",
              "      <td>0.003431</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-96.566350</td>\n",
              "      <td>25.007096</td>\n",
              "      <td>0.272082</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>15</td>\n",
              "      <td>0.899864</td>\n",
              "      <td>0.004352</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.111279</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.007493</td>\n",
              "      <td>0.294222</td>\n",
              "      <td>0.295209</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-100.000000</td>\n",
              "      <td>11.127933</td>\n",
              "      <td>0.136195</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>16</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000585</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.002599</td>\n",
              "      <td>0.264759</td>\n",
              "      <td>0.265908</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-100.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    group  cumulative_data_fraction  ...  cumulative_gain  kolmogorov_smirnov\n",
              "0       1                  0.010218  ...       277.701544            0.038593\n",
              "1       2                  0.020209  ...       269.213869            0.073996\n",
              "2       3                  0.030200  ...       269.181960            0.110566\n",
              "3       4                  0.040191  ...       264.898102            0.144802\n",
              "4       5                  0.050182  ...       265.738146            0.181372\n",
              "5       6                  0.100136  ...       246.868765            0.336223\n",
              "6       7                  0.150091  ...       230.274572            0.470079\n",
              "7       8                  0.200045  ...       210.821361            0.573606\n",
              "8       9                  0.299955  ...       171.053038            0.697841\n",
              "9      10                  0.400091  ...       127.221133            0.692290\n",
              "10     11                  0.500000  ...        93.481990            0.635724\n",
              "11     12                  0.599909  ...        64.833414            0.528999\n",
              "12     13                  0.700045  ...        42.357831            0.403302\n",
              "13     14                  0.799955  ...        25.007096            0.272082\n",
              "14     15                  0.899864  ...        11.127933            0.136195\n",
              "15     16                  1.000000  ...         0.000000            0.000000\n",
              "\n",
              "[16 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "ModelMetricsBinomial: xgboost\n",
            "** Reported on validation data. **\n",
            "\n",
            "MSE: 0.1454319479231641\n",
            "RMSE: 0.3813554089339289\n",
            "LogLoss: 0.44142340098488275\n",
            "Mean Per-Class Error: 0.2407015982497126\n",
            "AUC: 0.8350762042496385\n",
            "AUCPR: 0.6383583958445556\n",
            "Gini: 0.6701524084992769\n",
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.23507675528526306: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>580.0</td>\n",
              "      <td>221.0</td>\n",
              "      <td>0.2759</td>\n",
              "      <td>(221.0/801.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>64.0</td>\n",
              "      <td>239.0</td>\n",
              "      <td>0.2112</td>\n",
              "      <td>(64.0/303.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>644.0</td>\n",
              "      <td>460.0</td>\n",
              "      <td>0.2582</td>\n",
              "      <td>(285.0/1104.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             No    Yes   Error             Rate\n",
              "0     No  580.0  221.0  0.2759    (221.0/801.0)\n",
              "1    Yes   64.0  239.0  0.2112     (64.0/303.0)\n",
              "2  Total  644.0  460.0  0.2582   (285.0/1104.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Maximum Metrics: Maximum metrics at their respective thresholds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>threshold</th>\n",
              "      <th>value</th>\n",
              "      <th>idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>max f1</td>\n",
              "      <td>0.235077</td>\n",
              "      <td>0.626474</td>\n",
              "      <td>245.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>max f2</td>\n",
              "      <td>0.072505</td>\n",
              "      <td>0.758530</td>\n",
              "      <td>331.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>max f0point5</td>\n",
              "      <td>0.410793</td>\n",
              "      <td>0.614203</td>\n",
              "      <td>177.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>max accuracy</td>\n",
              "      <td>0.573676</td>\n",
              "      <td>0.788043</td>\n",
              "      <td>123.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>max precision</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>max recall</td>\n",
              "      <td>0.003221</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>max specificity</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>max absolute_mcc</td>\n",
              "      <td>0.410793</td>\n",
              "      <td>0.474439</td>\n",
              "      <td>177.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>max min_per_class_accuracy</td>\n",
              "      <td>0.273960</td>\n",
              "      <td>0.745875</td>\n",
              "      <td>229.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>max mean_per_class_accuracy</td>\n",
              "      <td>0.182972</td>\n",
              "      <td>0.759298</td>\n",
              "      <td>269.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>max tns</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>801.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>max fns</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>302.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>max fps</td>\n",
              "      <td>0.000776</td>\n",
              "      <td>801.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>max tps</td>\n",
              "      <td>0.003221</td>\n",
              "      <td>303.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>max tnr</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>max fnr</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>0.996700</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>max fpr</td>\n",
              "      <td>0.000776</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>max tpr</td>\n",
              "      <td>0.003221</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         metric  threshold       value    idx\n",
              "0                        max f1   0.235077    0.626474  245.0\n",
              "1                        max f2   0.072505    0.758530  331.0\n",
              "2                  max f0point5   0.410793    0.614203  177.0\n",
              "3                  max accuracy   0.573676    0.788043  123.0\n",
              "4                 max precision   0.969253    1.000000    0.0\n",
              "5                    max recall   0.003221    1.000000  394.0\n",
              "6               max specificity   0.969253    1.000000    0.0\n",
              "7              max absolute_mcc   0.410793    0.474439  177.0\n",
              "8    max min_per_class_accuracy   0.273960    0.745875  229.0\n",
              "9   max mean_per_class_accuracy   0.182972    0.759298  269.0\n",
              "10                      max tns   0.969253  801.000000    0.0\n",
              "11                      max fns   0.969253  302.000000    0.0\n",
              "12                      max fps   0.000776  801.000000  399.0\n",
              "13                      max tps   0.003221  303.000000  394.0\n",
              "14                      max tnr   0.969253    1.000000    0.0\n",
              "15                      max fnr   0.969253    0.996700    0.0\n",
              "16                      max fpr   0.000776    1.000000  399.0\n",
              "17                      max tpr   0.003221    1.000000  394.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Gains/Lift Table: Avg response rate: 27.45 %, avg score: 26.67 %\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>group</th>\n",
              "      <th>cumulative_data_fraction</th>\n",
              "      <th>lower_threshold</th>\n",
              "      <th>lift</th>\n",
              "      <th>cumulative_lift</th>\n",
              "      <th>response_rate</th>\n",
              "      <th>score</th>\n",
              "      <th>cumulative_response_rate</th>\n",
              "      <th>cumulative_score</th>\n",
              "      <th>capture_rate</th>\n",
              "      <th>cumulative_capture_rate</th>\n",
              "      <th>gain</th>\n",
              "      <th>cumulative_gain</th>\n",
              "      <th>kolmogorov_smirnov</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0.010870</td>\n",
              "      <td>0.910905</td>\n",
              "      <td>3.339934</td>\n",
              "      <td>3.339934</td>\n",
              "      <td>0.916667</td>\n",
              "      <td>0.934347</td>\n",
              "      <td>0.916667</td>\n",
              "      <td>0.934347</td>\n",
              "      <td>0.036304</td>\n",
              "      <td>0.036304</td>\n",
              "      <td>233.993399</td>\n",
              "      <td>233.993399</td>\n",
              "      <td>0.035055</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0.020833</td>\n",
              "      <td>0.886614</td>\n",
              "      <td>2.649865</td>\n",
              "      <td>3.009901</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.894043</td>\n",
              "      <td>0.826087</td>\n",
              "      <td>0.915071</td>\n",
              "      <td>0.026403</td>\n",
              "      <td>0.062706</td>\n",
              "      <td>164.986499</td>\n",
              "      <td>200.990099</td>\n",
              "      <td>0.057713</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>0.030797</td>\n",
              "      <td>0.867023</td>\n",
              "      <td>2.649865</td>\n",
              "      <td>2.893419</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.875754</td>\n",
              "      <td>0.794118</td>\n",
              "      <td>0.902351</td>\n",
              "      <td>0.026403</td>\n",
              "      <td>0.089109</td>\n",
              "      <td>164.986499</td>\n",
              "      <td>189.341875</td>\n",
              "      <td>0.080370</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>0.040761</td>\n",
              "      <td>0.853073</td>\n",
              "      <td>2.649865</td>\n",
              "      <td>2.833883</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.861681</td>\n",
              "      <td>0.777778</td>\n",
              "      <td>0.892409</td>\n",
              "      <td>0.026403</td>\n",
              "      <td>0.115512</td>\n",
              "      <td>164.986499</td>\n",
              "      <td>183.388339</td>\n",
              "      <td>0.103027</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>0.050725</td>\n",
              "      <td>0.834888</td>\n",
              "      <td>2.981098</td>\n",
              "      <td>2.862801</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.842712</td>\n",
              "      <td>0.785714</td>\n",
              "      <td>0.882647</td>\n",
              "      <td>0.029703</td>\n",
              "      <td>0.145215</td>\n",
              "      <td>198.109811</td>\n",
              "      <td>186.280057</td>\n",
              "      <td>0.130233</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>6</td>\n",
              "      <td>0.100543</td>\n",
              "      <td>0.708878</td>\n",
              "      <td>2.649865</td>\n",
              "      <td>2.757292</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.774918</td>\n",
              "      <td>0.756757</td>\n",
              "      <td>0.829268</td>\n",
              "      <td>0.132013</td>\n",
              "      <td>0.277228</td>\n",
              "      <td>164.986499</td>\n",
              "      <td>175.729195</td>\n",
              "      <td>0.243520</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>7</td>\n",
              "      <td>0.150362</td>\n",
              "      <td>0.627603</td>\n",
              "      <td>1.788659</td>\n",
              "      <td>2.436359</td>\n",
              "      <td>0.490909</td>\n",
              "      <td>0.665935</td>\n",
              "      <td>0.668675</td>\n",
              "      <td>0.775152</td>\n",
              "      <td>0.089109</td>\n",
              "      <td>0.366337</td>\n",
              "      <td>78.865887</td>\n",
              "      <td>143.635930</td>\n",
              "      <td>0.297672</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>8</td>\n",
              "      <td>0.200181</td>\n",
              "      <td>0.556613</td>\n",
              "      <td>2.119892</td>\n",
              "      <td>2.357600</td>\n",
              "      <td>0.581818</td>\n",
              "      <td>0.591467</td>\n",
              "      <td>0.647059</td>\n",
              "      <td>0.729438</td>\n",
              "      <td>0.105611</td>\n",
              "      <td>0.471947</td>\n",
              "      <td>111.989199</td>\n",
              "      <td>135.760047</td>\n",
              "      <td>0.374569</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>9</td>\n",
              "      <td>0.299819</td>\n",
              "      <td>0.390592</td>\n",
              "      <td>1.788659</td>\n",
              "      <td>2.168526</td>\n",
              "      <td>0.490909</td>\n",
              "      <td>0.471205</td>\n",
              "      <td>0.595166</td>\n",
              "      <td>0.643621</td>\n",
              "      <td>0.178218</td>\n",
              "      <td>0.650165</td>\n",
              "      <td>78.865887</td>\n",
              "      <td>116.852622</td>\n",
              "      <td>0.482874</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>10</td>\n",
              "      <td>0.400362</td>\n",
              "      <td>0.259730</td>\n",
              "      <td>1.116047</td>\n",
              "      <td>1.904216</td>\n",
              "      <td>0.306306</td>\n",
              "      <td>0.324027</td>\n",
              "      <td>0.522624</td>\n",
              "      <td>0.563361</td>\n",
              "      <td>0.112211</td>\n",
              "      <td>0.762376</td>\n",
              "      <td>11.604674</td>\n",
              "      <td>90.421576</td>\n",
              "      <td>0.498956</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>11</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.152304</td>\n",
              "      <td>0.960576</td>\n",
              "      <td>1.716172</td>\n",
              "      <td>0.263636</td>\n",
              "      <td>0.199585</td>\n",
              "      <td>0.471014</td>\n",
              "      <td>0.490869</td>\n",
              "      <td>0.095710</td>\n",
              "      <td>0.858086</td>\n",
              "      <td>-3.942394</td>\n",
              "      <td>71.617162</td>\n",
              "      <td>0.493541</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>12</td>\n",
              "      <td>0.599638</td>\n",
              "      <td>0.085389</td>\n",
              "      <td>0.728713</td>\n",
              "      <td>1.552092</td>\n",
              "      <td>0.200000</td>\n",
              "      <td>0.115374</td>\n",
              "      <td>0.425982</td>\n",
              "      <td>0.428476</td>\n",
              "      <td>0.072607</td>\n",
              "      <td>0.930693</td>\n",
              "      <td>-27.128713</td>\n",
              "      <td>55.209237</td>\n",
              "      <td>0.456286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>13</td>\n",
              "      <td>0.700181</td>\n",
              "      <td>0.038123</td>\n",
              "      <td>0.426724</td>\n",
              "      <td>1.390494</td>\n",
              "      <td>0.117117</td>\n",
              "      <td>0.063148</td>\n",
              "      <td>0.381630</td>\n",
              "      <td>0.376016</td>\n",
              "      <td>0.042904</td>\n",
              "      <td>0.973597</td>\n",
              "      <td>-57.327625</td>\n",
              "      <td>39.049351</td>\n",
              "      <td>0.376843</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>14</td>\n",
              "      <td>0.799819</td>\n",
              "      <td>0.012180</td>\n",
              "      <td>0.231863</td>\n",
              "      <td>1.246157</td>\n",
              "      <td>0.063636</td>\n",
              "      <td>0.023983</td>\n",
              "      <td>0.342016</td>\n",
              "      <td>0.332161</td>\n",
              "      <td>0.023102</td>\n",
              "      <td>0.996700</td>\n",
              "      <td>-76.813681</td>\n",
              "      <td>24.615678</td>\n",
              "      <td>0.271356</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>15</td>\n",
              "      <td>0.899457</td>\n",
              "      <td>0.004520</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.108113</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.007714</td>\n",
              "      <td>0.304129</td>\n",
              "      <td>0.296221</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.996700</td>\n",
              "      <td>-100.000000</td>\n",
              "      <td>10.811323</td>\n",
              "      <td>0.134028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>16</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000631</td>\n",
              "      <td>0.032825</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.009009</td>\n",
              "      <td>0.002524</td>\n",
              "      <td>0.274457</td>\n",
              "      <td>0.266691</td>\n",
              "      <td>0.003300</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-96.717510</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    group  cumulative_data_fraction  ...  cumulative_gain  kolmogorov_smirnov\n",
              "0       1                  0.010870  ...       233.993399            0.035055\n",
              "1       2                  0.020833  ...       200.990099            0.057713\n",
              "2       3                  0.030797  ...       189.341875            0.080370\n",
              "3       4                  0.040761  ...       183.388339            0.103027\n",
              "4       5                  0.050725  ...       186.280057            0.130233\n",
              "5       6                  0.100543  ...       175.729195            0.243520\n",
              "6       7                  0.150362  ...       143.635930            0.297672\n",
              "7       8                  0.200181  ...       135.760047            0.374569\n",
              "8       9                  0.299819  ...       116.852622            0.482874\n",
              "9      10                  0.400362  ...        90.421576            0.498956\n",
              "10     11                  0.500000  ...        71.617162            0.493541\n",
              "11     12                  0.599638  ...        55.209237            0.456286\n",
              "12     13                  0.700181  ...        39.049351            0.376843\n",
              "13     14                  0.799819  ...        24.615678            0.271356\n",
              "14     15                  0.899457  ...        10.811323            0.134028\n",
              "15     16                  1.000000  ...         0.000000            0.000000\n",
              "\n",
              "[16 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Scoring History: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>timestamp</th>\n",
              "      <th>duration</th>\n",
              "      <th>number_of_trees</th>\n",
              "      <th>training_rmse</th>\n",
              "      <th>training_logloss</th>\n",
              "      <th>training_auc</th>\n",
              "      <th>training_pr_auc</th>\n",
              "      <th>training_lift</th>\n",
              "      <th>training_classification_error</th>\n",
              "      <th>validation_rmse</th>\n",
              "      <th>validation_logloss</th>\n",
              "      <th>validation_auc</th>\n",
              "      <th>validation_pr_auc</th>\n",
              "      <th>validation_lift</th>\n",
              "      <th>validation_classification_error</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.004 sec</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.693147</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.264759</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.735241</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.693147</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.274457</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.725543</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.152 sec</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.353899</td>\n",
              "      <td>0.405977</td>\n",
              "      <td>0.888560</td>\n",
              "      <td>0.737234</td>\n",
              "      <td>3.554838</td>\n",
              "      <td>0.192098</td>\n",
              "      <td>0.378375</td>\n",
              "      <td>0.447842</td>\n",
              "      <td>0.842425</td>\n",
              "      <td>0.656759</td>\n",
              "      <td>3.339934</td>\n",
              "      <td>0.252717</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.241 sec</td>\n",
              "      <td>10.0</td>\n",
              "      <td>0.338894</td>\n",
              "      <td>0.361790</td>\n",
              "      <td>0.895385</td>\n",
              "      <td>0.751796</td>\n",
              "      <td>3.702956</td>\n",
              "      <td>0.179382</td>\n",
              "      <td>0.373030</td>\n",
              "      <td>0.425332</td>\n",
              "      <td>0.843910</td>\n",
              "      <td>0.652443</td>\n",
              "      <td>3.036304</td>\n",
              "      <td>0.236413</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.328 sec</td>\n",
              "      <td>15.0</td>\n",
              "      <td>0.329747</td>\n",
              "      <td>0.341435</td>\n",
              "      <td>0.905554</td>\n",
              "      <td>0.774180</td>\n",
              "      <td>3.609148</td>\n",
              "      <td>0.163715</td>\n",
              "      <td>0.374630</td>\n",
              "      <td>0.426870</td>\n",
              "      <td>0.841819</td>\n",
              "      <td>0.655824</td>\n",
              "      <td>3.643564</td>\n",
              "      <td>0.229167</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.416 sec</td>\n",
              "      <td>20.0</td>\n",
              "      <td>0.322682</td>\n",
              "      <td>0.327591</td>\n",
              "      <td>0.913443</td>\n",
              "      <td>0.792022</td>\n",
              "      <td>3.609148</td>\n",
              "      <td>0.155767</td>\n",
              "      <td>0.375006</td>\n",
              "      <td>0.427630</td>\n",
              "      <td>0.843475</td>\n",
              "      <td>0.656891</td>\n",
              "      <td>3.643564</td>\n",
              "      <td>0.237319</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.511 sec</td>\n",
              "      <td>25.0</td>\n",
              "      <td>0.318691</td>\n",
              "      <td>0.319265</td>\n",
              "      <td>0.917618</td>\n",
              "      <td>0.800432</td>\n",
              "      <td>3.609148</td>\n",
              "      <td>0.145322</td>\n",
              "      <td>0.376675</td>\n",
              "      <td>0.431292</td>\n",
              "      <td>0.841751</td>\n",
              "      <td>0.653366</td>\n",
              "      <td>3.643564</td>\n",
              "      <td>0.240942</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.617 sec</td>\n",
              "      <td>30.0</td>\n",
              "      <td>0.312266</td>\n",
              "      <td>0.308686</td>\n",
              "      <td>0.924772</td>\n",
              "      <td>0.815700</td>\n",
              "      <td>3.777015</td>\n",
              "      <td>0.136694</td>\n",
              "      <td>0.380153</td>\n",
              "      <td>0.438176</td>\n",
              "      <td>0.836601</td>\n",
              "      <td>0.642911</td>\n",
              "      <td>3.643564</td>\n",
              "      <td>0.260870</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:30</td>\n",
              "      <td>0.750 sec</td>\n",
              "      <td>35.0</td>\n",
              "      <td>0.306587</td>\n",
              "      <td>0.299498</td>\n",
              "      <td>0.930594</td>\n",
              "      <td>0.829947</td>\n",
              "      <td>3.777015</td>\n",
              "      <td>0.135786</td>\n",
              "      <td>0.381355</td>\n",
              "      <td>0.441423</td>\n",
              "      <td>0.835076</td>\n",
              "      <td>0.638358</td>\n",
              "      <td>3.339934</td>\n",
              "      <td>0.258152</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               timestamp  ... validation_lift  validation_classification_error\n",
              "0    2020-10-20 09:01:29  ...        1.000000                         0.725543\n",
              "1    2020-10-20 09:01:29  ...        3.339934                         0.252717\n",
              "2    2020-10-20 09:01:29  ...        3.036304                         0.236413\n",
              "3    2020-10-20 09:01:29  ...        3.643564                         0.229167\n",
              "4    2020-10-20 09:01:29  ...        3.643564                         0.237319\n",
              "5    2020-10-20 09:01:29  ...        3.643564                         0.240942\n",
              "6    2020-10-20 09:01:29  ...        3.643564                         0.260870\n",
              "7    2020-10-20 09:01:30  ...        3.339934                         0.258152\n",
              "\n",
              "[8 rows x 16 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Variable Importances: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>variable</th>\n",
              "      <th>relative_importance</th>\n",
              "      <th>scaled_importance</th>\n",
              "      <th>percentage</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Contract.Month-to-month</td>\n",
              "      <td>774.048157</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.184802</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Total_Charges</td>\n",
              "      <td>620.412842</td>\n",
              "      <td>0.801517</td>\n",
              "      <td>0.148122</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Monthly_Charges</td>\n",
              "      <td>478.453613</td>\n",
              "      <td>0.618119</td>\n",
              "      <td>0.114230</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Tenure_Months</td>\n",
              "      <td>457.393982</td>\n",
              "      <td>0.590912</td>\n",
              "      <td>0.109202</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Internet_Service.Fiber optic</td>\n",
              "      <td>301.830078</td>\n",
              "      <td>0.389937</td>\n",
              "      <td>0.072061</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Online_Security.No</td>\n",
              "      <td>276.856628</td>\n",
              "      <td>0.357674</td>\n",
              "      <td>0.066099</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Contract.Two year</td>\n",
              "      <td>161.932205</td>\n",
              "      <td>0.209202</td>\n",
              "      <td>0.038661</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Dependents.No</td>\n",
              "      <td>130.228760</td>\n",
              "      <td>0.168244</td>\n",
              "      <td>0.031092</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Dependents.Yes</td>\n",
              "      <td>119.304550</td>\n",
              "      <td>0.154131</td>\n",
              "      <td>0.028484</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Payment_Method.Electronic check</td>\n",
              "      <td>87.307266</td>\n",
              "      <td>0.112793</td>\n",
              "      <td>0.020844</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Tech_Support.No</td>\n",
              "      <td>72.298958</td>\n",
              "      <td>0.093404</td>\n",
              "      <td>0.017261</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Online_Backup.No</td>\n",
              "      <td>63.203594</td>\n",
              "      <td>0.081653</td>\n",
              "      <td>0.015090</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Gender.Female</td>\n",
              "      <td>62.815063</td>\n",
              "      <td>0.081151</td>\n",
              "      <td>0.014997</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>Internet_Service.DSL</td>\n",
              "      <td>58.047939</td>\n",
              "      <td>0.074993</td>\n",
              "      <td>0.013859</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Paperless_Billing.No</td>\n",
              "      <td>57.018730</td>\n",
              "      <td>0.073663</td>\n",
              "      <td>0.013613</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>Senior_Citizen.No</td>\n",
              "      <td>36.171650</td>\n",
              "      <td>0.046730</td>\n",
              "      <td>0.008636</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Partner.No</td>\n",
              "      <td>35.257767</td>\n",
              "      <td>0.045550</td>\n",
              "      <td>0.008418</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>Streaming_Movies.Yes</td>\n",
              "      <td>34.656902</td>\n",
              "      <td>0.044774</td>\n",
              "      <td>0.008274</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>Multiple_Lines.No</td>\n",
              "      <td>29.436729</td>\n",
              "      <td>0.038030</td>\n",
              "      <td>0.007028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>Partner.Yes</td>\n",
              "      <td>27.257126</td>\n",
              "      <td>0.035214</td>\n",
              "      <td>0.006508</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                           variable  ...  percentage\n",
              "0           Contract.Month-to-month  ...    0.184802\n",
              "1                     Total_Charges  ...    0.148122\n",
              "2                   Monthly_Charges  ...    0.114230\n",
              "3                     Tenure_Months  ...    0.109202\n",
              "4      Internet_Service.Fiber optic  ...    0.072061\n",
              "5                Online_Security.No  ...    0.066099\n",
              "6                 Contract.Two year  ...    0.038661\n",
              "7                     Dependents.No  ...    0.031092\n",
              "8                    Dependents.Yes  ...    0.028484\n",
              "9   Payment_Method.Electronic check  ...    0.020844\n",
              "10                  Tech_Support.No  ...    0.017261\n",
              "11                 Online_Backup.No  ...    0.015090\n",
              "12                    Gender.Female  ...    0.014997\n",
              "13             Internet_Service.DSL  ...    0.013859\n",
              "14             Paperless_Billing.No  ...    0.013613\n",
              "15                Senior_Citizen.No  ...    0.008636\n",
              "16                       Partner.No  ...    0.008418\n",
              "17             Streaming_Movies.Yes  ...    0.008274\n",
              "18                Multiple_Lines.No  ...    0.007028\n",
              "19                      Partner.Yes  ...    0.006508\n",
              "\n",
              "[20 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "See the whole table with table.as_data_frame()\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yssPHqdORr5n"
      },
      "source": [
        "#XGBoost"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "85ppw0hgMxxv"
      },
      "source": [
        "out = h2o.get_model([mid for mid in model_ids if \"XGBoost\" in mid][0])"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b4f7YNB6hqbq",
        "outputId": "13dcd6a6-208e-409a-d471-35fe05f238d0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "out.params"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'backend': {'actual': 'gpu', 'default': 'auto', 'input': 'auto'},\n",
              " 'booster': {'actual': 'gbtree', 'default': 'gbtree', 'input': 'gbtree'},\n",
              " 'build_tree_one_node': {'actual': False, 'default': False, 'input': False},\n",
              " 'calibrate_model': {'actual': False, 'default': False, 'input': False},\n",
              " 'calibration_frame': {'actual': None, 'default': None, 'input': None},\n",
              " 'categorical_encoding': {'actual': 'OneHotInternal',\n",
              "  'default': 'AUTO',\n",
              "  'input': 'AUTO'},\n",
              " 'checkpoint': {'actual': None, 'default': None, 'input': None},\n",
              " 'col_sample_rate': {'actual': 0.8, 'default': 1.0, 'input': 0.8},\n",
              " 'col_sample_rate_per_tree': {'actual': 0.8, 'default': 1.0, 'input': 0.8},\n",
              " 'colsample_bylevel': {'actual': 0.8, 'default': 1.0, 'input': 1.0},\n",
              " 'colsample_bynode': {'actual': 1.0, 'default': 1.0, 'input': 1.0},\n",
              " 'colsample_bytree': {'actual': 0.8, 'default': 1.0, 'input': 1.0},\n",
              " 'distribution': {'actual': 'bernoulli',\n",
              "  'default': 'AUTO',\n",
              "  'input': 'bernoulli'},\n",
              " 'dmatrix_type': {'actual': 'dense', 'default': 'auto', 'input': 'auto'},\n",
              " 'eta': {'actual': 0.3, 'default': 0.3, 'input': 0.3},\n",
              " 'export_checkpoints_dir': {'actual': None, 'default': None, 'input': None},\n",
              " 'fold_assignment': {'actual': None, 'default': 'AUTO', 'input': 'AUTO'},\n",
              " 'fold_column': {'actual': None, 'default': None, 'input': None},\n",
              " 'gainslift_bins': {'actual': -1, 'default': -1, 'input': -1},\n",
              " 'gamma': {'actual': 0.0, 'default': 0.0, 'input': 0.0},\n",
              " 'gpu_id': {'actual': 0, 'default': 0, 'input': 0},\n",
              " 'grow_policy': {'actual': 'depthwise',\n",
              "  'default': 'depthwise',\n",
              "  'input': 'depthwise'},\n",
              " 'ignore_const_cols': {'actual': True, 'default': True, 'input': True},\n",
              " 'ignored_columns': {'actual': [], 'default': None, 'input': []},\n",
              " 'keep_cross_validation_fold_assignment': {'actual': False,\n",
              "  'default': False,\n",
              "  'input': False},\n",
              " 'keep_cross_validation_models': {'actual': False,\n",
              "  'default': True,\n",
              "  'input': False},\n",
              " 'keep_cross_validation_predictions': {'actual': True,\n",
              "  'default': False,\n",
              "  'input': True},\n",
              " 'learn_rate': {'actual': 0.3, 'default': 0.3, 'input': 0.3},\n",
              " 'max_abs_leafnode_pred': {'actual': 0.0, 'default': 0.0, 'input': 0.0},\n",
              " 'max_bins': {'actual': 256, 'default': 256, 'input': 256},\n",
              " 'max_delta_step': {'actual': 0.0, 'default': 0.0, 'input': 0.0},\n",
              " 'max_depth': {'actual': 5, 'default': 6, 'input': 5},\n",
              " 'max_leaves': {'actual': 0, 'default': 0, 'input': 0},\n",
              " 'max_runtime_secs': {'actual': 368934873227853.8,\n",
              "  'default': 0.0,\n",
              "  'input': 368934873227853.8},\n",
              " 'min_child_weight': {'actual': 3.0, 'default': 1.0, 'input': 1.0},\n",
              " 'min_rows': {'actual': 3.0, 'default': 1.0, 'input': 3.0},\n",
              " 'min_split_improvement': {'actual': 0.0, 'default': 0.0, 'input': 0.0},\n",
              " 'model_id': {'actual': {'URL': '/3/Models/XGBoost_3_AutoML_20201020_090123',\n",
              "   '__meta': {'schema_name': 'ModelKeyV3',\n",
              "    'schema_type': 'Key<Model>',\n",
              "    'schema_version': 3},\n",
              "   'name': 'XGBoost_3_AutoML_20201020_090123',\n",
              "   'type': 'Key<Model>'},\n",
              "  'default': None,\n",
              "  'input': None},\n",
              " 'monotone_constraints': {'actual': None, 'default': None, 'input': None},\n",
              " 'nfolds': {'actual': 0, 'default': 0, 'input': 0},\n",
              " 'normalize_type': {'actual': 'tree', 'default': 'tree', 'input': 'tree'},\n",
              " 'nthread': {'actual': -1, 'default': -1, 'input': -1},\n",
              " 'ntrees': {'actual': 10000, 'default': 50, 'input': 10000},\n",
              " 'offset_column': {'actual': None, 'default': None, 'input': None},\n",
              " 'one_drop': {'actual': False, 'default': False, 'input': False},\n",
              " 'quiet_mode': {'actual': True, 'default': True, 'input': True},\n",
              " 'rate_drop': {'actual': 0.0, 'default': 0.0, 'input': 0.0},\n",
              " 'reg_alpha': {'actual': 0.0, 'default': 0.0, 'input': 0.0},\n",
              " 'reg_lambda': {'actual': 1.0, 'default': 1.0, 'input': 1.0},\n",
              " 'response_column': {'actual': {'__meta': {'schema_name': 'ColSpecifierV3',\n",
              "    'schema_type': 'VecSpecifier',\n",
              "    'schema_version': 3},\n",
              "   'column_name': 'Churn_Label',\n",
              "   'is_member_of_frames': None},\n",
              "  'default': None,\n",
              "  'input': {'__meta': {'schema_name': 'ColSpecifierV3',\n",
              "    'schema_type': 'VecSpecifier',\n",
              "    'schema_version': 3},\n",
              "   'column_name': 'Churn_Label',\n",
              "   'is_member_of_frames': None}},\n",
              " 'sample_rate': {'actual': 0.8, 'default': 1.0, 'input': 0.8},\n",
              " 'sample_type': {'actual': 'uniform',\n",
              "  'default': 'uniform',\n",
              "  'input': 'uniform'},\n",
              " 'save_matrix_directory': {'actual': None, 'default': None, 'input': None},\n",
              " 'score_each_iteration': {'actual': False, 'default': False, 'input': False},\n",
              " 'score_tree_interval': {'actual': 5, 'default': 0, 'input': 5},\n",
              " 'seed': {'actual': 12, 'default': -1, 'input': 12},\n",
              " 'skip_drop': {'actual': 0.0, 'default': 0.0, 'input': 0.0},\n",
              " 'stopping_metric': {'actual': 'logloss',\n",
              "  'default': 'AUTO',\n",
              "  'input': 'logloss'},\n",
              " 'stopping_rounds': {'actual': 3, 'default': 0, 'input': 3},\n",
              " 'stopping_tolerance': {'actual': 0.01432082875315512,\n",
              "  'default': 0.001,\n",
              "  'input': 0.01432082875315512},\n",
              " 'subsample': {'actual': 0.8, 'default': 1.0, 'input': 1.0},\n",
              " 'training_frame': {'actual': {'URL': '/3/Frames/automl_training_py_2_sid_8e46',\n",
              "   '__meta': {'schema_name': 'FrameKeyV3',\n",
              "    'schema_type': 'Key<Frame>',\n",
              "    'schema_version': 3},\n",
              "   'name': 'automl_training_py_2_sid_8e46',\n",
              "   'type': 'Key<Frame>'},\n",
              "  'default': None,\n",
              "  'input': {'URL': '/3/Frames/automl_training_py_2_sid_8e46',\n",
              "   '__meta': {'schema_name': 'FrameKeyV3',\n",
              "    'schema_type': 'Key<Frame>',\n",
              "    'schema_version': 3},\n",
              "   'name': 'automl_training_py_2_sid_8e46',\n",
              "   'type': 'Key<Frame>'}},\n",
              " 'tree_method': {'actual': 'hist', 'default': 'auto', 'input': 'auto'},\n",
              " 'tweedie_power': {'actual': 1.5, 'default': 1.5, 'input': 1.5},\n",
              " 'validation_frame': {'actual': {'URL': '/3/Frames/py_4_sid_8e46',\n",
              "   '__meta': {'schema_name': 'FrameKeyV3',\n",
              "    'schema_type': 'Key<Frame>',\n",
              "    'schema_version': 3},\n",
              "   'name': 'py_4_sid_8e46',\n",
              "   'type': 'Key<Frame>'},\n",
              "  'default': None,\n",
              "  'input': {'URL': '/3/Frames/py_4_sid_8e46',\n",
              "   '__meta': {'schema_name': 'FrameKeyV3',\n",
              "    'schema_type': 'Key<Frame>',\n",
              "    'schema_version': 3},\n",
              "   'name': 'py_4_sid_8e46',\n",
              "   'type': 'Key<Frame>'}},\n",
              " 'weights_column': {'actual': None, 'default': None, 'input': None}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mxY-GjDVRxVl",
        "outputId": "f12435eb-b91a-487b-9e52-1d1a3a38fc0e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        }
      },
      "source": [
        "#Convert H2OXGBoost parameter into XGBoost Parameters\n",
        "out.convert_H2OXGBoostParams_2_XGBoostParams()"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'alpha': 0.0,\n",
              "  'booster': 'gbtree',\n",
              "  'colsample_bylevel': 0.8,\n",
              "  'colsample_bytree': 0.8,\n",
              "  'eta': 0.3,\n",
              "  'gamma': 0.0,\n",
              "  'gpu_id': 0,\n",
              "  'grow_policy': 'depthwise',\n",
              "  'lambda': 1.0,\n",
              "  'max_bin': 256,\n",
              "  'max_delta_step': 0.0,\n",
              "  'max_depth': 5,\n",
              "  'min_child_weight': 3.0,\n",
              "  'nround': 10000,\n",
              "  'nthread': 2,\n",
              "  'objective': 'binary:logistic',\n",
              "  'seed': 12,\n",
              "  'silent': True,\n",
              "  'subsample': 0.8,\n",
              "  'updater': 'grow_gpu_hist'},\n",
              " 10000)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wl4mI1K9lteZ",
        "outputId": "95b50921-289d-49a6-8284-7346524206e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "out"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model Details\n",
            "=============\n",
            "H2OXGBoostEstimator :  XGBoost\n",
            "Model Key:  XGBoost_3_AutoML_20201020_090123\n",
            "\n",
            "\n",
            "Model Summary: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>number_of_trees</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>35.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     number_of_trees\n",
              "0               35.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "ModelMetricsBinomial: xgboost\n",
            "** Reported on train data. **\n",
            "\n",
            "MSE: 0.09399571002559544\n",
            "RMSE: 0.30658719807845114\n",
            "LogLoss: 0.29949752998721063\n",
            "Mean Per-Class Error: 0.1452000631438206\n",
            "AUC: 0.9305935784005755\n",
            "AUCPR: 0.8299469678919649\n",
            "Gini: 0.8611871568011511\n",
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.3705852244581495: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>2841.0</td>\n",
              "      <td>397.0</td>\n",
              "      <td>0.1226</td>\n",
              "      <td>(397.0/3238.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>201.0</td>\n",
              "      <td>965.0</td>\n",
              "      <td>0.1724</td>\n",
              "      <td>(201.0/1166.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>3042.0</td>\n",
              "      <td>1362.0</td>\n",
              "      <td>0.1358</td>\n",
              "      <td>(598.0/4404.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              No     Yes   Error             Rate\n",
              "0     No  2841.0   397.0  0.1226   (397.0/3238.0)\n",
              "1    Yes   201.0   965.0  0.1724   (201.0/1166.0)\n",
              "2  Total  3042.0  1362.0  0.1358   (598.0/4404.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Maximum Metrics: Maximum metrics at their respective thresholds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>threshold</th>\n",
              "      <th>value</th>\n",
              "      <th>idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>max f1</td>\n",
              "      <td>0.370585</td>\n",
              "      <td>0.763449</td>\n",
              "      <td>208.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>max f2</td>\n",
              "      <td>0.197192</td>\n",
              "      <td>0.829819</td>\n",
              "      <td>279.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>max f0point5</td>\n",
              "      <td>0.572713</td>\n",
              "      <td>0.774095</td>\n",
              "      <td>133.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>max accuracy</td>\n",
              "      <td>0.456477</td>\n",
              "      <td>0.870345</td>\n",
              "      <td>176.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>max precision</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>max recall</td>\n",
              "      <td>0.017452</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>379.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>max specificity</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>max absolute_mcc</td>\n",
              "      <td>0.370585</td>\n",
              "      <td>0.672999</td>\n",
              "      <td>208.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>max min_per_class_accuracy</td>\n",
              "      <td>0.337709</td>\n",
              "      <td>0.854540</td>\n",
              "      <td>222.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>max mean_per_class_accuracy</td>\n",
              "      <td>0.337709</td>\n",
              "      <td>0.854800</td>\n",
              "      <td>222.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>max tns</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>3238.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>max fns</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>1165.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>max fps</td>\n",
              "      <td>0.001026</td>\n",
              "      <td>3238.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>max tps</td>\n",
              "      <td>0.017452</td>\n",
              "      <td>1166.000000</td>\n",
              "      <td>379.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>max tnr</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>max fnr</td>\n",
              "      <td>0.986969</td>\n",
              "      <td>0.999142</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>max fpr</td>\n",
              "      <td>0.001026</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>max tpr</td>\n",
              "      <td>0.017452</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>379.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         metric  threshold        value    idx\n",
              "0                        max f1   0.370585     0.763449  208.0\n",
              "1                        max f2   0.197192     0.829819  279.0\n",
              "2                  max f0point5   0.572713     0.774095  133.0\n",
              "3                  max accuracy   0.456477     0.870345  176.0\n",
              "4                 max precision   0.986969     1.000000    0.0\n",
              "5                    max recall   0.017452     1.000000  379.0\n",
              "6               max specificity   0.986969     1.000000    0.0\n",
              "7              max absolute_mcc   0.370585     0.672999  208.0\n",
              "8    max min_per_class_accuracy   0.337709     0.854540  222.0\n",
              "9   max mean_per_class_accuracy   0.337709     0.854800  222.0\n",
              "10                      max tns   0.986969  3238.000000    0.0\n",
              "11                      max fns   0.986969  1165.000000    0.0\n",
              "12                      max fps   0.001026  3238.000000  399.0\n",
              "13                      max tps   0.017452  1166.000000  379.0\n",
              "14                      max tnr   0.986969     1.000000    0.0\n",
              "15                      max fnr   0.986969     0.999142    0.0\n",
              "16                      max fpr   0.001026     1.000000  399.0\n",
              "17                      max tpr   0.017452     1.000000  379.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Gains/Lift Table: Avg response rate: 26.48 %, avg score: 26.59 %\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>group</th>\n",
              "      <th>cumulative_data_fraction</th>\n",
              "      <th>lower_threshold</th>\n",
              "      <th>lift</th>\n",
              "      <th>cumulative_lift</th>\n",
              "      <th>response_rate</th>\n",
              "      <th>score</th>\n",
              "      <th>cumulative_response_rate</th>\n",
              "      <th>cumulative_score</th>\n",
              "      <th>capture_rate</th>\n",
              "      <th>cumulative_capture_rate</th>\n",
              "      <th>gain</th>\n",
              "      <th>cumulative_gain</th>\n",
              "      <th>kolmogorov_smirnov</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0.010218</td>\n",
              "      <td>0.931747</td>\n",
              "      <td>3.777015</td>\n",
              "      <td>3.777015</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.956726</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.956726</td>\n",
              "      <td>0.038593</td>\n",
              "      <td>0.038593</td>\n",
              "      <td>277.701544</td>\n",
              "      <td>277.701544</td>\n",
              "      <td>0.038593</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0.020209</td>\n",
              "      <td>0.907316</td>\n",
              "      <td>3.605333</td>\n",
              "      <td>3.692139</td>\n",
              "      <td>0.954545</td>\n",
              "      <td>0.918276</td>\n",
              "      <td>0.977528</td>\n",
              "      <td>0.937717</td>\n",
              "      <td>0.036021</td>\n",
              "      <td>0.074614</td>\n",
              "      <td>260.533292</td>\n",
              "      <td>269.213869</td>\n",
              "      <td>0.073996</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>0.030200</td>\n",
              "      <td>0.889492</td>\n",
              "      <td>3.691174</td>\n",
              "      <td>3.691820</td>\n",
              "      <td>0.977273</td>\n",
              "      <td>0.899013</td>\n",
              "      <td>0.977444</td>\n",
              "      <td>0.924913</td>\n",
              "      <td>0.036878</td>\n",
              "      <td>0.111492</td>\n",
              "      <td>269.117418</td>\n",
              "      <td>269.181960</td>\n",
              "      <td>0.110566</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>0.040191</td>\n",
              "      <td>0.860961</td>\n",
              "      <td>3.519492</td>\n",
              "      <td>3.648981</td>\n",
              "      <td>0.931818</td>\n",
              "      <td>0.875097</td>\n",
              "      <td>0.966102</td>\n",
              "      <td>0.912529</td>\n",
              "      <td>0.035163</td>\n",
              "      <td>0.146655</td>\n",
              "      <td>251.949166</td>\n",
              "      <td>264.898102</td>\n",
              "      <td>0.144802</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>0.050182</td>\n",
              "      <td>0.840925</td>\n",
              "      <td>3.691174</td>\n",
              "      <td>3.657381</td>\n",
              "      <td>0.977273</td>\n",
              "      <td>0.850945</td>\n",
              "      <td>0.968326</td>\n",
              "      <td>0.900268</td>\n",
              "      <td>0.036878</td>\n",
              "      <td>0.183533</td>\n",
              "      <td>269.117418</td>\n",
              "      <td>265.738146</td>\n",
              "      <td>0.181372</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>6</td>\n",
              "      <td>0.100136</td>\n",
              "      <td>0.745977</td>\n",
              "      <td>3.279136</td>\n",
              "      <td>3.468688</td>\n",
              "      <td>0.868182</td>\n",
              "      <td>0.792359</td>\n",
              "      <td>0.918367</td>\n",
              "      <td>0.846436</td>\n",
              "      <td>0.163808</td>\n",
              "      <td>0.347341</td>\n",
              "      <td>227.913613</td>\n",
              "      <td>246.868765</td>\n",
              "      <td>0.336223</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>7</td>\n",
              "      <td>0.150091</td>\n",
              "      <td>0.647722</td>\n",
              "      <td>2.970108</td>\n",
              "      <td>3.302746</td>\n",
              "      <td>0.786364</td>\n",
              "      <td>0.695603</td>\n",
              "      <td>0.874433</td>\n",
              "      <td>0.796234</td>\n",
              "      <td>0.148370</td>\n",
              "      <td>0.495712</td>\n",
              "      <td>197.010759</td>\n",
              "      <td>230.274572</td>\n",
              "      <td>0.470079</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>8</td>\n",
              "      <td>0.200045</td>\n",
              "      <td>0.558343</td>\n",
              "      <td>2.523733</td>\n",
              "      <td>3.108214</td>\n",
              "      <td>0.668182</td>\n",
              "      <td>0.605589</td>\n",
              "      <td>0.822928</td>\n",
              "      <td>0.748627</td>\n",
              "      <td>0.126072</td>\n",
              "      <td>0.621784</td>\n",
              "      <td>152.373304</td>\n",
              "      <td>210.821361</td>\n",
              "      <td>0.573606</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>9</td>\n",
              "      <td>0.299955</td>\n",
              "      <td>0.380472</td>\n",
              "      <td>1.914260</td>\n",
              "      <td>2.710530</td>\n",
              "      <td>0.506818</td>\n",
              "      <td>0.466967</td>\n",
              "      <td>0.717638</td>\n",
              "      <td>0.654811</td>\n",
              "      <td>0.191252</td>\n",
              "      <td>0.813036</td>\n",
              "      <td>91.426010</td>\n",
              "      <td>171.053038</td>\n",
              "      <td>0.697841</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>10</td>\n",
              "      <td>0.400091</td>\n",
              "      <td>0.253055</td>\n",
              "      <td>0.959242</td>\n",
              "      <td>2.272211</td>\n",
              "      <td>0.253968</td>\n",
              "      <td>0.315455</td>\n",
              "      <td>0.601589</td>\n",
              "      <td>0.569876</td>\n",
              "      <td>0.096055</td>\n",
              "      <td>0.909091</td>\n",
              "      <td>-4.075798</td>\n",
              "      <td>127.221133</td>\n",
              "      <td>0.692290</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>11</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.144239</td>\n",
              "      <td>0.583721</td>\n",
              "      <td>1.934820</td>\n",
              "      <td>0.154545</td>\n",
              "      <td>0.196314</td>\n",
              "      <td>0.512262</td>\n",
              "      <td>0.495232</td>\n",
              "      <td>0.058319</td>\n",
              "      <td>0.967410</td>\n",
              "      <td>-41.627943</td>\n",
              "      <td>93.481990</td>\n",
              "      <td>0.635724</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>12</td>\n",
              "      <td>0.599909</td>\n",
              "      <td>0.070011</td>\n",
              "      <td>0.214603</td>\n",
              "      <td>1.648334</td>\n",
              "      <td>0.056818</td>\n",
              "      <td>0.104651</td>\n",
              "      <td>0.436412</td>\n",
              "      <td>0.430184</td>\n",
              "      <td>0.021441</td>\n",
              "      <td>0.988851</td>\n",
              "      <td>-78.539685</td>\n",
              "      <td>64.833414</td>\n",
              "      <td>0.528999</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>13</td>\n",
              "      <td>0.700045</td>\n",
              "      <td>0.030656</td>\n",
              "      <td>0.077082</td>\n",
              "      <td>1.423578</td>\n",
              "      <td>0.020408</td>\n",
              "      <td>0.047875</td>\n",
              "      <td>0.376906</td>\n",
              "      <td>0.375498</td>\n",
              "      <td>0.007719</td>\n",
              "      <td>0.996569</td>\n",
              "      <td>-92.291805</td>\n",
              "      <td>42.357831</td>\n",
              "      <td>0.403302</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>14</td>\n",
              "      <td>0.799955</td>\n",
              "      <td>0.012430</td>\n",
              "      <td>0.034337</td>\n",
              "      <td>1.250071</td>\n",
              "      <td>0.009091</td>\n",
              "      <td>0.020357</td>\n",
              "      <td>0.330968</td>\n",
              "      <td>0.331143</td>\n",
              "      <td>0.003431</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-96.566350</td>\n",
              "      <td>25.007096</td>\n",
              "      <td>0.272082</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>15</td>\n",
              "      <td>0.899864</td>\n",
              "      <td>0.004352</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.111279</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.007493</td>\n",
              "      <td>0.294222</td>\n",
              "      <td>0.295209</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-100.000000</td>\n",
              "      <td>11.127933</td>\n",
              "      <td>0.136195</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>16</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000585</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.002599</td>\n",
              "      <td>0.264759</td>\n",
              "      <td>0.265908</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-100.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    group  cumulative_data_fraction  ...  cumulative_gain  kolmogorov_smirnov\n",
              "0       1                  0.010218  ...       277.701544            0.038593\n",
              "1       2                  0.020209  ...       269.213869            0.073996\n",
              "2       3                  0.030200  ...       269.181960            0.110566\n",
              "3       4                  0.040191  ...       264.898102            0.144802\n",
              "4       5                  0.050182  ...       265.738146            0.181372\n",
              "5       6                  0.100136  ...       246.868765            0.336223\n",
              "6       7                  0.150091  ...       230.274572            0.470079\n",
              "7       8                  0.200045  ...       210.821361            0.573606\n",
              "8       9                  0.299955  ...       171.053038            0.697841\n",
              "9      10                  0.400091  ...       127.221133            0.692290\n",
              "10     11                  0.500000  ...        93.481990            0.635724\n",
              "11     12                  0.599909  ...        64.833414            0.528999\n",
              "12     13                  0.700045  ...        42.357831            0.403302\n",
              "13     14                  0.799955  ...        25.007096            0.272082\n",
              "14     15                  0.899864  ...        11.127933            0.136195\n",
              "15     16                  1.000000  ...         0.000000            0.000000\n",
              "\n",
              "[16 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "ModelMetricsBinomial: xgboost\n",
            "** Reported on validation data. **\n",
            "\n",
            "MSE: 0.1454319479231641\n",
            "RMSE: 0.3813554089339289\n",
            "LogLoss: 0.44142340098488275\n",
            "Mean Per-Class Error: 0.2407015982497126\n",
            "AUC: 0.8350762042496385\n",
            "AUCPR: 0.6383583958445556\n",
            "Gini: 0.6701524084992769\n",
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.23507675528526306: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>580.0</td>\n",
              "      <td>221.0</td>\n",
              "      <td>0.2759</td>\n",
              "      <td>(221.0/801.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>64.0</td>\n",
              "      <td>239.0</td>\n",
              "      <td>0.2112</td>\n",
              "      <td>(64.0/303.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>644.0</td>\n",
              "      <td>460.0</td>\n",
              "      <td>0.2582</td>\n",
              "      <td>(285.0/1104.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             No    Yes   Error             Rate\n",
              "0     No  580.0  221.0  0.2759    (221.0/801.0)\n",
              "1    Yes   64.0  239.0  0.2112     (64.0/303.0)\n",
              "2  Total  644.0  460.0  0.2582   (285.0/1104.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Maximum Metrics: Maximum metrics at their respective thresholds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>threshold</th>\n",
              "      <th>value</th>\n",
              "      <th>idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>max f1</td>\n",
              "      <td>0.235077</td>\n",
              "      <td>0.626474</td>\n",
              "      <td>245.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>max f2</td>\n",
              "      <td>0.072505</td>\n",
              "      <td>0.758530</td>\n",
              "      <td>331.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>max f0point5</td>\n",
              "      <td>0.410793</td>\n",
              "      <td>0.614203</td>\n",
              "      <td>177.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>max accuracy</td>\n",
              "      <td>0.573676</td>\n",
              "      <td>0.788043</td>\n",
              "      <td>123.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>max precision</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>max recall</td>\n",
              "      <td>0.003221</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>max specificity</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>max absolute_mcc</td>\n",
              "      <td>0.410793</td>\n",
              "      <td>0.474439</td>\n",
              "      <td>177.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>max min_per_class_accuracy</td>\n",
              "      <td>0.273960</td>\n",
              "      <td>0.745875</td>\n",
              "      <td>229.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>max mean_per_class_accuracy</td>\n",
              "      <td>0.182972</td>\n",
              "      <td>0.759298</td>\n",
              "      <td>269.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>max tns</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>801.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>max fns</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>302.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>max fps</td>\n",
              "      <td>0.000776</td>\n",
              "      <td>801.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>max tps</td>\n",
              "      <td>0.003221</td>\n",
              "      <td>303.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>max tnr</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>max fnr</td>\n",
              "      <td>0.969253</td>\n",
              "      <td>0.996700</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>max fpr</td>\n",
              "      <td>0.000776</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>max tpr</td>\n",
              "      <td>0.003221</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         metric  threshold       value    idx\n",
              "0                        max f1   0.235077    0.626474  245.0\n",
              "1                        max f2   0.072505    0.758530  331.0\n",
              "2                  max f0point5   0.410793    0.614203  177.0\n",
              "3                  max accuracy   0.573676    0.788043  123.0\n",
              "4                 max precision   0.969253    1.000000    0.0\n",
              "5                    max recall   0.003221    1.000000  394.0\n",
              "6               max specificity   0.969253    1.000000    0.0\n",
              "7              max absolute_mcc   0.410793    0.474439  177.0\n",
              "8    max min_per_class_accuracy   0.273960    0.745875  229.0\n",
              "9   max mean_per_class_accuracy   0.182972    0.759298  269.0\n",
              "10                      max tns   0.969253  801.000000    0.0\n",
              "11                      max fns   0.969253  302.000000    0.0\n",
              "12                      max fps   0.000776  801.000000  399.0\n",
              "13                      max tps   0.003221  303.000000  394.0\n",
              "14                      max tnr   0.969253    1.000000    0.0\n",
              "15                      max fnr   0.969253    0.996700    0.0\n",
              "16                      max fpr   0.000776    1.000000  399.0\n",
              "17                      max tpr   0.003221    1.000000  394.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Gains/Lift Table: Avg response rate: 27.45 %, avg score: 26.67 %\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>group</th>\n",
              "      <th>cumulative_data_fraction</th>\n",
              "      <th>lower_threshold</th>\n",
              "      <th>lift</th>\n",
              "      <th>cumulative_lift</th>\n",
              "      <th>response_rate</th>\n",
              "      <th>score</th>\n",
              "      <th>cumulative_response_rate</th>\n",
              "      <th>cumulative_score</th>\n",
              "      <th>capture_rate</th>\n",
              "      <th>cumulative_capture_rate</th>\n",
              "      <th>gain</th>\n",
              "      <th>cumulative_gain</th>\n",
              "      <th>kolmogorov_smirnov</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0.010870</td>\n",
              "      <td>0.910905</td>\n",
              "      <td>3.339934</td>\n",
              "      <td>3.339934</td>\n",
              "      <td>0.916667</td>\n",
              "      <td>0.934347</td>\n",
              "      <td>0.916667</td>\n",
              "      <td>0.934347</td>\n",
              "      <td>0.036304</td>\n",
              "      <td>0.036304</td>\n",
              "      <td>233.993399</td>\n",
              "      <td>233.993399</td>\n",
              "      <td>0.035055</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0.020833</td>\n",
              "      <td>0.886614</td>\n",
              "      <td>2.649865</td>\n",
              "      <td>3.009901</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.894043</td>\n",
              "      <td>0.826087</td>\n",
              "      <td>0.915071</td>\n",
              "      <td>0.026403</td>\n",
              "      <td>0.062706</td>\n",
              "      <td>164.986499</td>\n",
              "      <td>200.990099</td>\n",
              "      <td>0.057713</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>0.030797</td>\n",
              "      <td>0.867023</td>\n",
              "      <td>2.649865</td>\n",
              "      <td>2.893419</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.875754</td>\n",
              "      <td>0.794118</td>\n",
              "      <td>0.902351</td>\n",
              "      <td>0.026403</td>\n",
              "      <td>0.089109</td>\n",
              "      <td>164.986499</td>\n",
              "      <td>189.341875</td>\n",
              "      <td>0.080370</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>0.040761</td>\n",
              "      <td>0.853073</td>\n",
              "      <td>2.649865</td>\n",
              "      <td>2.833883</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.861681</td>\n",
              "      <td>0.777778</td>\n",
              "      <td>0.892409</td>\n",
              "      <td>0.026403</td>\n",
              "      <td>0.115512</td>\n",
              "      <td>164.986499</td>\n",
              "      <td>183.388339</td>\n",
              "      <td>0.103027</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>0.050725</td>\n",
              "      <td>0.834888</td>\n",
              "      <td>2.981098</td>\n",
              "      <td>2.862801</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.842712</td>\n",
              "      <td>0.785714</td>\n",
              "      <td>0.882647</td>\n",
              "      <td>0.029703</td>\n",
              "      <td>0.145215</td>\n",
              "      <td>198.109811</td>\n",
              "      <td>186.280057</td>\n",
              "      <td>0.130233</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>6</td>\n",
              "      <td>0.100543</td>\n",
              "      <td>0.708878</td>\n",
              "      <td>2.649865</td>\n",
              "      <td>2.757292</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.774918</td>\n",
              "      <td>0.756757</td>\n",
              "      <td>0.829268</td>\n",
              "      <td>0.132013</td>\n",
              "      <td>0.277228</td>\n",
              "      <td>164.986499</td>\n",
              "      <td>175.729195</td>\n",
              "      <td>0.243520</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>7</td>\n",
              "      <td>0.150362</td>\n",
              "      <td>0.627603</td>\n",
              "      <td>1.788659</td>\n",
              "      <td>2.436359</td>\n",
              "      <td>0.490909</td>\n",
              "      <td>0.665935</td>\n",
              "      <td>0.668675</td>\n",
              "      <td>0.775152</td>\n",
              "      <td>0.089109</td>\n",
              "      <td>0.366337</td>\n",
              "      <td>78.865887</td>\n",
              "      <td>143.635930</td>\n",
              "      <td>0.297672</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>8</td>\n",
              "      <td>0.200181</td>\n",
              "      <td>0.556613</td>\n",
              "      <td>2.119892</td>\n",
              "      <td>2.357600</td>\n",
              "      <td>0.581818</td>\n",
              "      <td>0.591467</td>\n",
              "      <td>0.647059</td>\n",
              "      <td>0.729438</td>\n",
              "      <td>0.105611</td>\n",
              "      <td>0.471947</td>\n",
              "      <td>111.989199</td>\n",
              "      <td>135.760047</td>\n",
              "      <td>0.374569</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>9</td>\n",
              "      <td>0.299819</td>\n",
              "      <td>0.390592</td>\n",
              "      <td>1.788659</td>\n",
              "      <td>2.168526</td>\n",
              "      <td>0.490909</td>\n",
              "      <td>0.471205</td>\n",
              "      <td>0.595166</td>\n",
              "      <td>0.643621</td>\n",
              "      <td>0.178218</td>\n",
              "      <td>0.650165</td>\n",
              "      <td>78.865887</td>\n",
              "      <td>116.852622</td>\n",
              "      <td>0.482874</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>10</td>\n",
              "      <td>0.400362</td>\n",
              "      <td>0.259730</td>\n",
              "      <td>1.116047</td>\n",
              "      <td>1.904216</td>\n",
              "      <td>0.306306</td>\n",
              "      <td>0.324027</td>\n",
              "      <td>0.522624</td>\n",
              "      <td>0.563361</td>\n",
              "      <td>0.112211</td>\n",
              "      <td>0.762376</td>\n",
              "      <td>11.604674</td>\n",
              "      <td>90.421576</td>\n",
              "      <td>0.498956</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>11</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.152304</td>\n",
              "      <td>0.960576</td>\n",
              "      <td>1.716172</td>\n",
              "      <td>0.263636</td>\n",
              "      <td>0.199585</td>\n",
              "      <td>0.471014</td>\n",
              "      <td>0.490869</td>\n",
              "      <td>0.095710</td>\n",
              "      <td>0.858086</td>\n",
              "      <td>-3.942394</td>\n",
              "      <td>71.617162</td>\n",
              "      <td>0.493541</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>12</td>\n",
              "      <td>0.599638</td>\n",
              "      <td>0.085389</td>\n",
              "      <td>0.728713</td>\n",
              "      <td>1.552092</td>\n",
              "      <td>0.200000</td>\n",
              "      <td>0.115374</td>\n",
              "      <td>0.425982</td>\n",
              "      <td>0.428476</td>\n",
              "      <td>0.072607</td>\n",
              "      <td>0.930693</td>\n",
              "      <td>-27.128713</td>\n",
              "      <td>55.209237</td>\n",
              "      <td>0.456286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>13</td>\n",
              "      <td>0.700181</td>\n",
              "      <td>0.038123</td>\n",
              "      <td>0.426724</td>\n",
              "      <td>1.390494</td>\n",
              "      <td>0.117117</td>\n",
              "      <td>0.063148</td>\n",
              "      <td>0.381630</td>\n",
              "      <td>0.376016</td>\n",
              "      <td>0.042904</td>\n",
              "      <td>0.973597</td>\n",
              "      <td>-57.327625</td>\n",
              "      <td>39.049351</td>\n",
              "      <td>0.376843</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>14</td>\n",
              "      <td>0.799819</td>\n",
              "      <td>0.012180</td>\n",
              "      <td>0.231863</td>\n",
              "      <td>1.246157</td>\n",
              "      <td>0.063636</td>\n",
              "      <td>0.023983</td>\n",
              "      <td>0.342016</td>\n",
              "      <td>0.332161</td>\n",
              "      <td>0.023102</td>\n",
              "      <td>0.996700</td>\n",
              "      <td>-76.813681</td>\n",
              "      <td>24.615678</td>\n",
              "      <td>0.271356</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>15</td>\n",
              "      <td>0.899457</td>\n",
              "      <td>0.004520</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.108113</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.007714</td>\n",
              "      <td>0.304129</td>\n",
              "      <td>0.296221</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.996700</td>\n",
              "      <td>-100.000000</td>\n",
              "      <td>10.811323</td>\n",
              "      <td>0.134028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>16</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000631</td>\n",
              "      <td>0.032825</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.009009</td>\n",
              "      <td>0.002524</td>\n",
              "      <td>0.274457</td>\n",
              "      <td>0.266691</td>\n",
              "      <td>0.003300</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-96.717510</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    group  cumulative_data_fraction  ...  cumulative_gain  kolmogorov_smirnov\n",
              "0       1                  0.010870  ...       233.993399            0.035055\n",
              "1       2                  0.020833  ...       200.990099            0.057713\n",
              "2       3                  0.030797  ...       189.341875            0.080370\n",
              "3       4                  0.040761  ...       183.388339            0.103027\n",
              "4       5                  0.050725  ...       186.280057            0.130233\n",
              "5       6                  0.100543  ...       175.729195            0.243520\n",
              "6       7                  0.150362  ...       143.635930            0.297672\n",
              "7       8                  0.200181  ...       135.760047            0.374569\n",
              "8       9                  0.299819  ...       116.852622            0.482874\n",
              "9      10                  0.400362  ...        90.421576            0.498956\n",
              "10     11                  0.500000  ...        71.617162            0.493541\n",
              "11     12                  0.599638  ...        55.209237            0.456286\n",
              "12     13                  0.700181  ...        39.049351            0.376843\n",
              "13     14                  0.799819  ...        24.615678            0.271356\n",
              "14     15                  0.899457  ...        10.811323            0.134028\n",
              "15     16                  1.000000  ...         0.000000            0.000000\n",
              "\n",
              "[16 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Scoring History: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>timestamp</th>\n",
              "      <th>duration</th>\n",
              "      <th>number_of_trees</th>\n",
              "      <th>training_rmse</th>\n",
              "      <th>training_logloss</th>\n",
              "      <th>training_auc</th>\n",
              "      <th>training_pr_auc</th>\n",
              "      <th>training_lift</th>\n",
              "      <th>training_classification_error</th>\n",
              "      <th>validation_rmse</th>\n",
              "      <th>validation_logloss</th>\n",
              "      <th>validation_auc</th>\n",
              "      <th>validation_pr_auc</th>\n",
              "      <th>validation_lift</th>\n",
              "      <th>validation_classification_error</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.004 sec</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.693147</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.264759</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.735241</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.693147</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.274457</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.725543</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.152 sec</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.353899</td>\n",
              "      <td>0.405977</td>\n",
              "      <td>0.888560</td>\n",
              "      <td>0.737234</td>\n",
              "      <td>3.554838</td>\n",
              "      <td>0.192098</td>\n",
              "      <td>0.378375</td>\n",
              "      <td>0.447842</td>\n",
              "      <td>0.842425</td>\n",
              "      <td>0.656759</td>\n",
              "      <td>3.339934</td>\n",
              "      <td>0.252717</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.241 sec</td>\n",
              "      <td>10.0</td>\n",
              "      <td>0.338894</td>\n",
              "      <td>0.361790</td>\n",
              "      <td>0.895385</td>\n",
              "      <td>0.751796</td>\n",
              "      <td>3.702956</td>\n",
              "      <td>0.179382</td>\n",
              "      <td>0.373030</td>\n",
              "      <td>0.425332</td>\n",
              "      <td>0.843910</td>\n",
              "      <td>0.652443</td>\n",
              "      <td>3.036304</td>\n",
              "      <td>0.236413</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.328 sec</td>\n",
              "      <td>15.0</td>\n",
              "      <td>0.329747</td>\n",
              "      <td>0.341435</td>\n",
              "      <td>0.905554</td>\n",
              "      <td>0.774180</td>\n",
              "      <td>3.609148</td>\n",
              "      <td>0.163715</td>\n",
              "      <td>0.374630</td>\n",
              "      <td>0.426870</td>\n",
              "      <td>0.841819</td>\n",
              "      <td>0.655824</td>\n",
              "      <td>3.643564</td>\n",
              "      <td>0.229167</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.416 sec</td>\n",
              "      <td>20.0</td>\n",
              "      <td>0.322682</td>\n",
              "      <td>0.327591</td>\n",
              "      <td>0.913443</td>\n",
              "      <td>0.792022</td>\n",
              "      <td>3.609148</td>\n",
              "      <td>0.155767</td>\n",
              "      <td>0.375006</td>\n",
              "      <td>0.427630</td>\n",
              "      <td>0.843475</td>\n",
              "      <td>0.656891</td>\n",
              "      <td>3.643564</td>\n",
              "      <td>0.237319</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.511 sec</td>\n",
              "      <td>25.0</td>\n",
              "      <td>0.318691</td>\n",
              "      <td>0.319265</td>\n",
              "      <td>0.917618</td>\n",
              "      <td>0.800432</td>\n",
              "      <td>3.609148</td>\n",
              "      <td>0.145322</td>\n",
              "      <td>0.376675</td>\n",
              "      <td>0.431292</td>\n",
              "      <td>0.841751</td>\n",
              "      <td>0.653366</td>\n",
              "      <td>3.643564</td>\n",
              "      <td>0.240942</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:29</td>\n",
              "      <td>0.617 sec</td>\n",
              "      <td>30.0</td>\n",
              "      <td>0.312266</td>\n",
              "      <td>0.308686</td>\n",
              "      <td>0.924772</td>\n",
              "      <td>0.815700</td>\n",
              "      <td>3.777015</td>\n",
              "      <td>0.136694</td>\n",
              "      <td>0.380153</td>\n",
              "      <td>0.438176</td>\n",
              "      <td>0.836601</td>\n",
              "      <td>0.642911</td>\n",
              "      <td>3.643564</td>\n",
              "      <td>0.260870</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td></td>\n",
              "      <td>2020-10-20 09:01:30</td>\n",
              "      <td>0.750 sec</td>\n",
              "      <td>35.0</td>\n",
              "      <td>0.306587</td>\n",
              "      <td>0.299498</td>\n",
              "      <td>0.930594</td>\n",
              "      <td>0.829947</td>\n",
              "      <td>3.777015</td>\n",
              "      <td>0.135786</td>\n",
              "      <td>0.381355</td>\n",
              "      <td>0.441423</td>\n",
              "      <td>0.835076</td>\n",
              "      <td>0.638358</td>\n",
              "      <td>3.339934</td>\n",
              "      <td>0.258152</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               timestamp  ... validation_lift  validation_classification_error\n",
              "0    2020-10-20 09:01:29  ...        1.000000                         0.725543\n",
              "1    2020-10-20 09:01:29  ...        3.339934                         0.252717\n",
              "2    2020-10-20 09:01:29  ...        3.036304                         0.236413\n",
              "3    2020-10-20 09:01:29  ...        3.643564                         0.229167\n",
              "4    2020-10-20 09:01:29  ...        3.643564                         0.237319\n",
              "5    2020-10-20 09:01:29  ...        3.643564                         0.240942\n",
              "6    2020-10-20 09:01:29  ...        3.643564                         0.260870\n",
              "7    2020-10-20 09:01:30  ...        3.339934                         0.258152\n",
              "\n",
              "[8 rows x 16 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Variable Importances: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>variable</th>\n",
              "      <th>relative_importance</th>\n",
              "      <th>scaled_importance</th>\n",
              "      <th>percentage</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Contract.Month-to-month</td>\n",
              "      <td>774.048157</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.184802</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Total_Charges</td>\n",
              "      <td>620.412842</td>\n",
              "      <td>0.801517</td>\n",
              "      <td>0.148122</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Monthly_Charges</td>\n",
              "      <td>478.453613</td>\n",
              "      <td>0.618119</td>\n",
              "      <td>0.114230</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Tenure_Months</td>\n",
              "      <td>457.393982</td>\n",
              "      <td>0.590912</td>\n",
              "      <td>0.109202</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Internet_Service.Fiber optic</td>\n",
              "      <td>301.830078</td>\n",
              "      <td>0.389937</td>\n",
              "      <td>0.072061</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Online_Security.No</td>\n",
              "      <td>276.856628</td>\n",
              "      <td>0.357674</td>\n",
              "      <td>0.066099</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Contract.Two year</td>\n",
              "      <td>161.932205</td>\n",
              "      <td>0.209202</td>\n",
              "      <td>0.038661</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Dependents.No</td>\n",
              "      <td>130.228760</td>\n",
              "      <td>0.168244</td>\n",
              "      <td>0.031092</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Dependents.Yes</td>\n",
              "      <td>119.304550</td>\n",
              "      <td>0.154131</td>\n",
              "      <td>0.028484</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Payment_Method.Electronic check</td>\n",
              "      <td>87.307266</td>\n",
              "      <td>0.112793</td>\n",
              "      <td>0.020844</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Tech_Support.No</td>\n",
              "      <td>72.298958</td>\n",
              "      <td>0.093404</td>\n",
              "      <td>0.017261</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Online_Backup.No</td>\n",
              "      <td>63.203594</td>\n",
              "      <td>0.081653</td>\n",
              "      <td>0.015090</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Gender.Female</td>\n",
              "      <td>62.815063</td>\n",
              "      <td>0.081151</td>\n",
              "      <td>0.014997</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>Internet_Service.DSL</td>\n",
              "      <td>58.047939</td>\n",
              "      <td>0.074993</td>\n",
              "      <td>0.013859</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Paperless_Billing.No</td>\n",
              "      <td>57.018730</td>\n",
              "      <td>0.073663</td>\n",
              "      <td>0.013613</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>Senior_Citizen.No</td>\n",
              "      <td>36.171650</td>\n",
              "      <td>0.046730</td>\n",
              "      <td>0.008636</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Partner.No</td>\n",
              "      <td>35.257767</td>\n",
              "      <td>0.045550</td>\n",
              "      <td>0.008418</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>Streaming_Movies.Yes</td>\n",
              "      <td>34.656902</td>\n",
              "      <td>0.044774</td>\n",
              "      <td>0.008274</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>Multiple_Lines.No</td>\n",
              "      <td>29.436729</td>\n",
              "      <td>0.038030</td>\n",
              "      <td>0.007028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>Partner.Yes</td>\n",
              "      <td>27.257126</td>\n",
              "      <td>0.035214</td>\n",
              "      <td>0.006508</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                           variable  ...  percentage\n",
              "0           Contract.Month-to-month  ...    0.184802\n",
              "1                     Total_Charges  ...    0.148122\n",
              "2                   Monthly_Charges  ...    0.114230\n",
              "3                     Tenure_Months  ...    0.109202\n",
              "4      Internet_Service.Fiber optic  ...    0.072061\n",
              "5                Online_Security.No  ...    0.066099\n",
              "6                 Contract.Two year  ...    0.038661\n",
              "7                     Dependents.No  ...    0.031092\n",
              "8                    Dependents.Yes  ...    0.028484\n",
              "9   Payment_Method.Electronic check  ...    0.020844\n",
              "10                  Tech_Support.No  ...    0.017261\n",
              "11                 Online_Backup.No  ...    0.015090\n",
              "12                    Gender.Female  ...    0.014997\n",
              "13             Internet_Service.DSL  ...    0.013859\n",
              "14             Paperless_Billing.No  ...    0.013613\n",
              "15                Senior_Citizen.No  ...    0.008636\n",
              "16                       Partner.No  ...    0.008418\n",
              "17             Streaming_Movies.Yes  ...    0.008274\n",
              "18                Multiple_Lines.No  ...    0.007028\n",
              "19                      Partner.Yes  ...    0.006508\n",
              "\n",
              "[20 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "See the whole table with table.as_data_frame()\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ma9VlXcJRJD1"
      },
      "source": [
        "#GBM Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JGqA2fD7fJS7"
      },
      "source": [
        "out_gbm = h2o.get_model([mid for mid in model_ids if \"GBM\" in mid][0])"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zGGLvJlrM0r1",
        "outputId": "b20d0334-4167-47bc-a61b-e61dd4e410b8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        }
      },
      "source": [
        "out.confusion_matrix()"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.3705852244581495: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>2841.0</td>\n",
              "      <td>397.0</td>\n",
              "      <td>0.1226</td>\n",
              "      <td>(397.0/3238.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>201.0</td>\n",
              "      <td>965.0</td>\n",
              "      <td>0.1724</td>\n",
              "      <td>(201.0/1166.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>3042.0</td>\n",
              "      <td>1362.0</td>\n",
              "      <td>0.1358</td>\n",
              "      <td>(598.0/4404.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              No     Yes   Error             Rate\n",
              "0     No  2841.0   397.0  0.1226   (397.0/3238.0)\n",
              "1    Yes   201.0   965.0  0.1724   (201.0/1166.0)\n",
              "2  Total  3042.0  1362.0  0.1358   (598.0/4404.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WzYCf-itPIf_",
        "outputId": "260586d3-f161-4088-ee7c-7b7e13d3a36d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        }
      },
      "source": [
        "#Variable important plot\n",
        "out.varimp_plot()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA80AAAJTCAYAAAAyrfRyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdebgcVZ3/8ffHhEUEUXEDdIyiqKxhdVBRcBudOK74Q3RU0HHDn4oz44jiaHD5iTIOiguKDgQFxX1FQQeIIsoSIBBQUJGg4sYiUVYhfH9/1LnQNF13ycJNwvv1PP3c7qpTp77VfW+efPqcqkpVIUmSJEmS7uhu012AJEmSJEmrKkOzJEmSJEk9DM2SJEmSJPUwNEuSJEmS1MPQLEmSJElSD0OzJEmSJEk9DM2SpNVakt2SVJK5y9nP3q2fvaewzby2zazl2bckSVp1GZolScskyTEtMO47ibbfa22fe2fUtqYY+EJg/nTXsrIty5cWdwVJ5rf3Zbdx2swbfu/SeXqSjyRZmOTPSW5IclGSDyV5wAT73TzJx5JcmOSaJNe2bT+e5JFTPIYt274vS3Kfnjb/0Y7hi+PU899Jzk5yVZKb2s/Tk/xXkh1GbDO39Tn4uCXJkiQ/TvK6JDOncix3trvK37+0qjM0S5KW1afaz38Zr1EbhX0K8HvgWyuhjjOARwMfXQl9S6urdYDvAq8CLgf+BzgMuAF4I3BukkeM2jDJG4CfAq+l+7v9BPBx4DLgNcAFrc2kVNUFwNuATVpfw/vbFng38LvW/+C6JHkn8DPgTUABXwA+ABwNXA+8HliQ5HU9JfwAOLA9/h/wdW77N+OoyR6HpLuuVfrbNUnSqquq5if5ObBdku2r6uyepq8AAhxZVTevhDquAy5c0f1Kq7mlwNuBj1fVn8cWJrkbXQB+NfDfwD8NbpTkpcCHgauA51bVD4fW70oXOj+c5M9V9dlJ1nMI8EzgBUleMrZdknWBY4C1gH2q6qqh7d4BzAV+A+xVVacOd5zk/sB+wIY9+55fVXOHtvk7ui8GXpTkgKpaPMnjkHQX5EizJGl5jI02v3LUyiQzgH3oRoc+3ZY9J8nRSX7epnxem+SsJG9o/6Ef7mNs6unDkrw+yXlJrh+bsth3TnOSHZJ8OMm5bRrnDUl+keSDSe493kElmdOmb17bprV+uW9Ubpw+HtO2+0OSvyX5TZJPJtlkKv309H3rVOYkT01ySptCe3mSI5Pcq7XbLsm32zFck+SbGXH+dW6bArxOkvckuSTJjUkuTvLOJGv31PHkJMe39/fG9pkelOQO4WVgH2sneUeb6ntj+3znA0e2pkcOTaed1bbfpG136sB7+rskn0uyxYj9zWrbz2vPj01yRfs9WJDkmeO8v3smOXHg92Zxks8n2XFE272SnJzk6tb2Z0nenmSdvv7vDFV1U1W9dzAwt+W3AO9qL3cbXJdkA+BD7eWLhgNz2/4U4MXt5YfaNpOpp4C9gSXAR1toBXgfsCXw0ar63lA9D6ML/n8DnjEqMLe+/1RVb6MbfZ6Uqvo1cFF7eb/h9e3fj68k+VP7Pb003dT0jUf1l2TjdNPZF7ffzcuTfDWjp42v3f69O7v9bV7XtvtGkqe0NnsnqbbJE4f+JuZO9jglrRiGZknS8jiK7j+0eyVZb8T6ZwCbAv9bVZe0ZQcB2wOnAx8BPgOsTze6Nd5UyQ/TTeFc1J6P/A/0gFcCL6T7j/GRdFNTfw/8K3DqOP/Zfx7dSNpv235+AjwfOC2TPJczyctbfc8ATqYLIgvoprIvGAgMy+tZwHF0028/AfyCLph8LcnfAz+im1X2P62efwK+nRFfTjRfBF5ON43+o3RfdswFvpIkQ8f4auD7wOPo3q9D6EYn3wL8eCy4j/AVYF/gx3TvyyJgHvCNtv4b3DaV9kDg6rb8CcD+7fVX2v5OA/YAzkg3xXeUh9BN4Z8FfJZuau9WwDeS7D50TEkyDzgW2Ab4atvPKcCudCOlg+2PAD4HPLzV9LH2HrwbOD5D58vmtnNs5/bUeme5qf0cnvmxB3Bv4IyqOqFv46o6HjgTuE/bZlJaUH09cE/gqCRPpZsq/jO635th+9D9/n65TfGeqP9Jz2RJ8mDgkcBfuS08j617Jt3v5z8B/0s3In8R3XT1BUkeOtT+oXR/3/sCFwMfBE4A5tD9LQx/QTOP7t+Wtej+/TsU+CGwNfD01mYh3e8/wKXc/m9i/mSPU9IKUlU+fPjw4cPHMj/oQkgBe49Y9422bo+BZZuNaHc3usBcwGOG1s1ryy8DHjpi293a+rlDyx8CzBjR/hWt/VuGlu/dlhfwzKF1b2zLT+ypbdbAss3pvkj4JbDpUPsn002b/dok39uxY5vfU+vNwBOH3sfvt3VXAS8e2u5/2rpnDy2f35b/HLj3wPJ16b40KOAlQ+/tjcBfgEcN9fXx1v7wnn2cB9x3xLGOHdMdfo/a+vsDG4xYvi1wDfDdoeWzBj7Pdw6t+4e2/DtDy1/Vlp8BbDi0bgaw8Yh6vwrcfajt3LbujT3L5446xp7jHnvf5rXtRz0WjvfejejzLa3953t+P947iT7e29p+erLHMrDtl9q2N9D9rWzf0+6k1u4VU93H0Ps9f+C9eg/dl2hXtsfzhrZZvy1fCuza8759b2j5CW35AUPLH0v3N3olsH5btiFwC13IHvXv00ZDr+/w9+/Dh487/zHtBfjw4cOHj9X7QRcEC/jR0PKN6Ua0/gisNYl+tm/9vGNo+bxRAWRg/W5TCSJ051cvAU4aWr43I4JxWzeDLgQX8JARtc0aWHZIWzanZ/9fa/+RvkMAHOfY5vfU+tkR27y0rfvhiHVPZHSInM9QMB5Rw8kDyw5oy/7fiPb3pgvT1wPrjNjHs4e3GTqmvZfhd/CbdAFsrYFls1p/i3vCyaXAFUPLFrVttpvEPs9pv9/36vl9uYJuxHZw+X2BRzHiS4Nx9jP2vk3mMeF7B+wEXNc+o82G1n2n9fOaSfTzGkZ88TDJY3roQM0HjdPup63N00esm8UdvzzYb6jN3HHeq5voZp88aGibF7f1nxuxz5nAJW3937VlD2qvL2XEv3N0sxsKeGl7fc/2+lQgk3ivDM0+fKwCDy8EJklaXifRTUl8XJJHV9XP2vKxqZXzqmpsOihJNgLeDPwj8DDgHkP9bdqznzOmUlSStegudvRCYAu6EZ7Bacl9+/nB8IKqWprkR8BmwHZ0/0Hus0v7+cQkO41Yf3+6ULU5cNZ4xzAJC0Ys+137Oarvy9rPB/X0d4djp5vivZTuuMds336eNNy4qv6c5By66dSPAs4dajKlz3FQkjl0YW1HugA6/P+Y+9JNwR+0sKqWjujuN9z2WZHkHnTTtv9YVedMUMd6dCPcVwD7Dc1cH3Mj3RWab1VVV7RtlsXuVTW/p555wMsm6iDJ5nRT79cCXlhVFy9jLctr7sDzZyZ5Z1XdOMU+ZgHvHFp2Kbedkz3owGoXAmunJmwMPIduGvVzkuxcVb9pbcf73b45yQ/bvrcDfs1tfxenDP47N+Ak4J9bu89U1V+SfItu6vfCJF+hm/5/enUXNZS0CjI0S5KWS1VVkk/TXdDnX4B/a+e/jk2DHrtYGO081zPpRprOoDuf7yq6kdd70U2D7ruA0h+mWNoXgOcCv6KbJv4HuiAD3ZV2+/bzxwn233eF3jEbtZ9vnqDd+hOsn4wlI5bdPIl1a/X0d4djb0HhCrqwP2bsPRgOqAwtH3Ve81Q/RwCSvJEuEP2Zbgr6r+lGTIsuAG3L6M/06hHLoHsvBr9EGav1shFth92bbsbC/bhjcFsltcB8Mt15yC+sqm+OaDb22Tx4El2OtfnduK3uWMfz6WZDnEY3rfw1dLeB+reeeh5Nd6uq22lfIKT1OZPbztMeV3UXQrsM+Fi7qNcBdBcbe3VrMtXf7WX5W9iTbqr3i7jtvOUbknwZ+Peq6vs3SNI0MTRLklaEI+muyPvSJG+lu2jSw+imQP9yoN2/0AXmW0d+xiTZhS4096nJFtOucvxcuov4PKMGLhDURpr+Y5zNH9Cz/IHt56gwOmhs/YZV9ZdJlLsqeQBdGL1VCyT3pZvOO2bsGB8IjLpA08ZD7W5VVZP+HIdqmEsXoravqt8Prd9l1HZTNBau+2YgDBo7rnOqavtxW64CkjwaOJHuC50XVNU3epr+iG6GyFPowuR4ntJ+TnRBvsE6NgY+CVwLvIQucD+ZbrT+G3XHq3WfCuze2hwx2f1Mwent584DywZ/t0cZ/t2eanuq6nralPJ2QbIn0J2e8M90o9i7Tli5pDuVV8+WJC23NjLyTbpw9Ry6cAxw+FDTh7efXxnRzRNXYElj+/lm3fGKujsDdx9n2zvUke7WWY9vL8eduks3ggar5398R30Gj6ebTj543GPPdxtu3GYTzKY7x/hnw+vHMTaFesaIdfelG6n78YjAvD63TaldZlV1LXA+8IAk203Q9hq6Lwu2THKf5d33ypRka7rzou9Dd9GrvsAM8GW6Lw92ble27uvzqXR/R1e1bSbrCLrg/m9V9cs2HXnsHPx5I65oP49uRsAeLfivaGO3nhv8//B4v9szue3v+uyh9o8fvlp6s/tQ+9upqt9U1TF0F6f7Zetno4EmtzD6b0LSncjQLElaUcamYf8b3SjvFXQXvRq0uP3cbXBhCylvXYG19O3n/nS3BRrPk0bcIub/0p3PfHJVjXc+M3S3aroJOKRNib2ddo/WVTVQ/2cG7mGdZF26afdw232UAY6mO8bXJ3k4t/duuosdHT3F81SvbD9H3Y7rT3RTsXdoIXmsvrXobt1z3ynsZzyHtp+fzNC9ppPcbegevf8NrA0cMer2WknunWT7oWX3TfKoJCuq3nElmU03JXsDuguwHTde+zYzYmya9OeSPG5En4+lu80WwJuq6q+TrGVfutspHVdVnxzY52nA++lmoBwyVM/FdFe7Xhv4btv3KH23NxuvnnXobhEFt7+F09fpvgzYq922bdB+rc7/re72WVTVb+lOF5jV1g/u4zF0U7D/TPu3MMn92hcZw+5Bd8rGzXRXFB9zJZObLi9pJXJ6tiRpRfkeXVgdm+r40ar621Cbz9Cd6/uhdo/cXwCPoLv/7VfpzvVbEc6km9r5vCQ/ppt2+gC6+yZfxPjnYX6L7j7HX6Mb+ZndtruK2/6T3auqLmz3aT4CuCDJ8XS3clqLLhDuSndf5Uct26GtVD+jq/nLdKH42XRfFhxHdxVgAKpqcZL96L6AODvJF+mO6Yl0F9e6kNH33R3PT+iC8X5tpG3s/NqPVNWSJIfS3ad5UZJv0AWp3elGUE/mthG95fFpus/nJcAv2n4upzun9kl0n+lcgKo6IskOtHvzJjmBbmr7feiC1RPovmh4zUD//5fuHOgDuf3FsFa49uXHia2eE4Fdeqaxf6iqbj3vux3XvYAPAKckmU93UbkCdqB7n2+hu1L1ZyZZy+bAwXRfpL1iRJO5dBcGfEWSr1fVtwfWvYvu3OX/pLu/+ll010O4ii4sz+K2qeLD07vH7DZwb+zQTZl+Bt0F8X5Fd/usseO/pv39fgn4QZIv0X2uOwBPo/u9fDW39xq6f28OTvI0ugv0PRh4Ad17tc/AlwubAuckWUR3+7Xf0H3J9Ey6Kd6HDn0RcSLwwnbxsLPp/i5/OGIqu6SVabov3+3Dhw8fPtacB7fdiqiAR/a02YJuKvef6M5tPItuOvestt28ofbzGLqt09D63Rh9n+b70N0zeDHdVOGL6S44tF5btnio/d6tn73p/gP7k1bf1XTTyTcfse/e2oCt2/pL6S5AdhXd9N9PAk+a5Ps5dmzz+2qd7PvR1vW9x/Pb8nXoRvYuaTX/ii7krdNT39Poviz5c2v/S7qwNeo2TPNppzSPc7xPb+/7NQO/R7PaupnAv9Ldhuh6uvDyWbp7Rt/hc+g71snUQ3fboR/QnYd6Q3s/jmHE/YTb78q36X6f/9bqOqO9j8P3sJ7b99mM856MfTa7jdNm7Pj3Hlg2dvwTPfr+rh5Fd0umi+i+zLiO7sufw4aPa4L6Z7b3o4DnjtNuq/Ze/57R9/F+JN1I9EK6v8mb6P6mzmzLR302Y+/38ONauqu6v2fU72rbdie60eHL2+f663bsm/S037Stv7S1v4Ju1HqnoXb3At5Bd1Xty+j+bn7fPue9GLoNFd0F+D5Hd5G+pVP9/fHhw8eKeaRqytfjkCRJa5A2mvjEqhp57yRJku7KPKdZkiRJkqQehmZJkiRJknoYmiVJkiRJ6uE5zZIkSZIk9fCWU1rjHXXUUfWyl71susuQJEmStOrqvRim07O1xrv22munuwRJkiRJqylDsyRJkiRJPQzNkiRJkiT1MDRLkiRJktTD0CxJkiRJUg9DsyRJkiRJPQzNkiRJkiT1MDRLkiRJktTD0CxJkiRJUg9DsyRJkiRJPQzNkiRJkiT1MDRLkiRJktTD0CxJkiRJUg9DsyRJkiRJPQzNkiRJkiT1MDRLkiRJktTD0CxJkiRJUg9DsyRJkiRJPQzNkiRJkiT1MDRLkiRJktTD0CxJkiRJUg9DsyRJkiRJPQzNkiRJkiT1MDRLkiRJktTD0CxJkiRJUg9DsyRJkiRJPQzNkiRJkiT1MDRLkiRJktTD0CxJkiRJUo+Z012AtLItumwJs/Y/brrLkCRJkgQsPmjOdJcwJY40S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9VgjQ3OSByY5NsnFSc5K8p0kmy9DP/slWW8F1TQryYvGWV9Jjh54PTPJ5Um+vYz7u1eSfQde7zaZviaq886W5DlJthh4PT/JjtNZkyRJkqS7jjUuNCcJ8DVgflVtVlU7AG8FHrAM3e0HjAzNSWZMsa9ZwHhh9FpgqyR3b6+fClw2xX0Muhew74St7mgW49d5Z3sOsMWErSRJkiRpJVjjQjOwO3BTVX1ibEFVnQv8KMnBSc5PsijJnnDrCOz8JF9OcmGSY9J5A7AJcHKSk1vba5J8MMm5wC5J3pHkzNbn4S2wk+ThSf43yblJzk6yGXAQsGuShUne1FP7d4A57flewOfHViS5T5KvJzkvyWlJtmnL5yY5oh3Dr1rdtP1t1vZ3cFu2/vBxjqjhdnUmWTfJke09OyfJ7qMKTzIvyWGttl+19/WIJD9LMm+g3V6tr/OTvH9g+TVJ3tves9OSPCDJY4FnAQe3ejZrzV+Q5IwkP0+ya897KUmSJEnLbU0MzVsBZ41Y/jxgNrAt8BS6ILZxW7cd3ajyFsDDgMdV1aHA74Ddq2osKN4DOL2qtq2qHwEfraqdqmor4O7AM1u7Y4CPVdW2wGOB3wP7A6dU1eyqOqSn9mOBFyZZF9gGOH1g3YHAOVW1DfA24DMD6x4F/AOwM/DOJGu1/V3c9vfmvuMcUcNwna8Dqqq2pgvyR7X6Rrk3sAvwJuCbwCHAlsDWSWYn2QR4P/Akus9ipyTPadveAzitvWc/BF5ZVT9u/by51XNxazuzqnZux/LOUYUkeVWSBUkWLL1uSU+5kiRJkjS+NTE093k88PmqWlpVfwR+AOzU1p1RVb+tqluAhXRTlEdZCnxl4PXuSU5PsoguCG6ZZANg06r6GkBV3VBV102mwKo6r+17L7pR5+H6P9vanQRslOSebd1xVXVjVV0B/In+qeiTPc7h/R7d9nshcCnQd374t6qqgEXAH6tqUdvXBW1fO9FNm7+8qm6m+3LhCW3bvwFj51yfNUFtX52oXVUdXlU7VtWOM9bbcKJjlCRJkqSR1sTQfAGwwxS3uXHg+VJgZk+7G6pqKUAbbf04sEcbhf0U0DcCOxXfBP6LganZkzDZ+u/QLslj2tTnhUmeNdkdtqnUC5MsHNH/LUP7umWcmsbc1AL3rbWN0/bGSbaTJEmSpOWyJobmk4B1krxqbEE7//dqYM8kM5Lcj26E84wJ+vorsEHPurGAfEWS9YE9AKrqr8Bvx6YdJ1kn3RW4x+tr0BHAgVW1aGj5KcCLW5+7AVdU1V+WsfZbVdXpberz7Kr65ojtBve7OfB3wEVVdcDYdpM4pjFnAE9Mct92IbW96Eb8xzPZ902SJEmSVrg1LjS30crnAk9Jd8upC4D3AZ8DzgPOpQvW/1FVf5igu8OB48cuBDa0n6vpRpfPB04AzhxY/RLgDUnOA34MPLDte2m70NWbkmySZHgKNm369KEjapkL7ND6PAh42XiFV9WVwKntglsHj9d2yO3qpBtNv1ubgv4FYO+qunHcHvprGju3+2S6z+GsqvrGBJsdC7y5XYRsswnaSpIkSdIKldtmxEprptce8L767tJtprsMSZIkScDig+ZM3OjON+rOQsAaONIsSZIkSdKKYmiWJEmSJKmHoVmSJEmSpB6GZkmSJEmSehiaJUmSJEnqYWiWJEmSJKmHoVmSJEmSpB6GZkmSJEmSehiaJUmSJEnqYWiWJEmSJKmHoVmSJEmSpB6GZkmSJEmSehiaJUmSJEnqMXO6C5BWtq033ZDD9p0z3WVIkiRJWg050ixJkiRJUg9DsyRJkiRJPQzNkiRJkiT1MDRLkiRJktTD0CxJkiRJUg9DsyRJkiRJPQzNkiRJkiT1MDRLkiRJktTD0CxJkiRJUo+Z012AtLItumwJs/Y/brrLkCRJWm0tPmjOdJcgTRtHmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQehmZJkiRJknoYmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQehmZJkiRJknoYmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQehmZJkiRJknoYmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQehmZJkiRJknoYmiVJkiRJ6mFoXg0l2SjJwvb4Q5LLBl6vPdR2vyTrTaLP+Ul2HGf9+kk+meTiJGe19o9JMivJ+SviuCRJkiRpVTNzugvQ1FXVlcBsgCRzgWuq6r96mu8HHA1ct5y7/TRwCfCIqrolyUOBLYA/Lk+nSWZW1c3LWZskSZIkrRSONK8hkjw5yTlJFiU5Isk6Sd4AbAKcnOTk1u6wJAuSXJDkwEn2vRnwGODtVXULQFVdUlXHtSYzknyq9fm9JHdv270yyZlJzk3ylbER7yTzknwiyenAB5JsluS0Vvt7klwzsO83tz7OG6s3yT2SHNf6PT/JniNqflU7zgVLr1uyrG+rJEmSpLs4Q/OaYV1gHrBnVW1NN4PgtVV1KPA7YPeq2r21PaCqdgS2AZ6YZJtJ9L8lsLCqlvasfwTwsaraErgaeH5b/tWq2qmqtgV+BrxiYJsHAY+tqn8FPgx8uNX+27EGSZ7W+t6ZbmR9hyRPAJ4O/K6qtq2qrYDjhwuqqsOraseq2nHGehtO4hAlSZIk6Y4MzWuGGcAlVfXz9voo4Ak9bf9PkrOBc+jC8BYrYP+XVNXC9vwsYFZ7vlWSU5IsAl7c9jfmSwMhfBfgS+355wbaPK09zgHOBh5FF6IXAU9N8v4ku1aVQ8mSJEmSVgrPab4Laech/zuwU1X9Ock8ulHqiVwAbJtkRs9o840Dz5cCd2/P5wHPqapzk+wN7DbQ7trJlAy8r6o+eYcVyfbAPwLvSXJiVb1rEv1JkiRJ0pQ40rxmWArMSvLw9volwA/a878CG7Tn96QLq0uSPAB4xmQ6r6qLgQXAgUkC0K6aPWeCTTcAfp9kLbqR5j6ncduU7hcOLD8BeHmS9ds+N01y/ySbANdV1dHAwcD2kzkOSZIkSZoqR5rXDDcA+wBfSjITOBP4RFt3OHB8kt9V1e5JzgEuBH4DnDqFffwL8EHgl0muB64A3jzBNv8JnA5c3n5u0NNuP+DoJAfQnZ+8BKCqvpfk0cBPWla/Bvhn4OHAwUluAW4CXjuF45AkSZKkSUtVTXcNuotrV9W+vqoqyQuBvarq2Suq/9ce8L767tLJXO9MkiRJoyw+aKIJhtJqL30rHGnWqmAH4KNt6vfVwMunuR5JkiRJAgzNGtLunbzO0OKXVNWilbXPqjoF2HZl9S9JkiRJy8rQrNupqsdMdw2SJEmStKrw6tmSJEmSJPUwNEuSJEmS1MPQLEmSJElSD0OzJEmSJEk9DM2SJEmSJPUwNEuSJEmS1MPQLEmSJElSD0OzJEmSJEk9DM2SJEmSJPUwNEuSJEmS1GPmdBcgrWxbb7ohh+07Z7rLkCRJkrQacqRZkiRJkqQehmZJkiRJknoYmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQehmZJkiRJknoYmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQeM6e7AGllW3TZEmbtf9x0lyFJ0ipt8UFzprsESVolOdIsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9A8DZJUkqMHXs9McnmSby9jf/dKsu/A6936+koyP8mOy7CPByY5NsnFSc5K8p0km4+3L0mSJEla3Rmap8e1wFZJ7t5ePxW4bDn6uxew74StllGSAF8D5lfVZlW1A/BW4AEroO+Zy9uHJEmSJK0shubp8x1gTnu+F/D5sRVJ7pPk60nOS3Jakm3a8rlJjmijxb9K8oa2yUHAZkkWJjm4LVs/yZeTXJjkmBZ8GdjHy5N8aOD1K5Mc0lPr7sBNVfWJsQVVdW5VnTLevpK8I8mZSc5PcvjA8vlJPpRkAfDGJDu1Y12Y5OAk57d2M9rrM9v6V7flGyf5YWt/fpJdhwtO8qokC5IsWHrdkok+C0mSJEkaydA8fY4FXphkXWAb4PSBdQcC51TVNsDbgM8MrHsU8A/AzsA7k6wF7A9cXFWzq+rNrd12wH7AFsDDgMcN7f+LwD+17QH2AY7oqXUr4KxxjovGqCIAACAASURBVKVvXx+tqp2qaivg7sAzB7ZZu6p2rKoPAkcCr66q2cDSgTavAJZU1U7ATsArkzwUeBFwQmu/LbBwuKCqOrz1v+OM9TYcp3RJkiRJ6mdoniZVdR4wi26U+TtDqx8PfLa1OwnYKMk927rjqurGqroC+BP9U6TPqKrfVtUtdKFy1tD+rwFOAp6Z5FHAWlW1aBkPp29fuyc5Pcki4EnAlgPbfAG687GBDarqJ2355wbaPA14aZKFdF8qbAQ8AjgT2CfJXGDrqvrrMtYtSZIkSePyfNLp9U3gv4Dd6ALhZNw48Hwp/Z/hZNp9mm4k+0K60d4+FwB7TKWmNoL+cWDHqvpNC7jrDrS7dpz+xgR4fVWdcIcVyRPoprfPS/LfVfWZO2wtSZIkScvJkebpdQRw4IgR3lOAF0N3JWzgiqr6yzj9/BXYYKo7r6rTgQfTTXf+/DhNTwLWSfKqsQVJthl1LvGAsYB8RZL16QndVXU18Nckj2mLXjiw+gTgtWNTyNvVuu+R5CHAH6vqU3TBf/tx6pAkSZKkZeZI8zSqqt8Ch45YNRc4Isl5wHXAyybo58okp7YLaH0XOG4KZXwRmF1Vfx6n/0ryXOBDSd4C3AAspjuPedOeba5O8ingfOAPdFOq+7wC+FSSW4AfAGNX7vo03VTvs9tFxC4HnkM3Mv/mJDcB1wAvndSRSpIkSdIUpaqmuwZNo3aP5UOq6sRprGH9do41SfYHNq6qN66o/l97wPvqu0u3WVHdSZK0Rlp80JyJG0nSmit9K5yefReV5F5Jfg5cP52BuZkzdvsoYFfgPdNcjyRJkiQBTs++y2rnEm8+uCzJRsCoAP3kqrpyJdbyBdrVtCVJkiRpVWJo1q1aMJ493XVIkiRJ0qrC6dmSJEmSJPUwNEuSJEmS1MPQLEmSJElSD0OzJEmSJEk9DM2SJEmSJPUwNEuSJEmS1MPQLEmSJElSD0OzJEmSJEk9DM2SJEmSJPUwNEuSJEmS1GPmdBcgrWxbb7ohh+07Z7rLkCRJkrQacqRZkiRJkqQehmZJkiRJknoYmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQehmZJkiRJknoYmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQeM6e7AGllW3TZEmbtf9x0lyFJ0gq1+KA5012CJN0lONIsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ/MqLMlGSRa2xx+SXDbweu1prm1xklOGli1Mcv5y9Pm2geezlqcvSZIkSVoRDM2rsKq6sqpmV9Vs4BPAIWOvq+pvK3JfSWYsw2YbJHlw2/7RK6CMt03cRJIkSZLuPIbm1UySHZL8IMlZSU5IsnFbPj/J+5OckeTnSXZty/dO8tGB7b+dZLf2/JokH0xyLrBLkn9u2y9M8slJBOkvAnu253sBnx/Yz7pJjkyyKMk5SXYfqOerSY5P8oskH2jLDwLu3vZ9TOtmRpJPJbkgyfeS3L21fUOSnyY5L8mxPe/Tq5IsSLJg6XVLpvIWS5IkSdKtDM2rlwAfAfaoqh2AI4D3DqyfWVU7A/sB75xEf/cATq+qbYEr6QLw49rI9lLgxRNs/xXgee35PwHfGlj3OqCqamu6QH1UknXbutltX1sDeyZ5cFXtD1zfRtHH9vsI4GNVtSVwNfD8tnx/YLuq2gZ4zajCqurwqtqxqnacsd6GE78TkiRJkjTCzOkuQFOyDrAV8P0kADOA3w+s/2r7eRYwaxL9LaULvgBPBnYAzmx93x340wTbXwn8OckLgZ8B1w2sezxdwKeqLkxyKbB5W3diVS0BSPJT4CHAb0b0f0lVLRxxTOcBxyT5OvD1iQ9TkiRJkpaNoXn1EuCCqtqlZ/2N7edSbvtsb+b2MwrWHXh+Q1UtHej7qKp66xRr+gLwMWDvKWxz48DzwVonanf39nwO8AS60e0DkmxdVTdPYf+SJEmSNClOz1693AjcL8kuAEnWSrLlBNssBmYnuVu7aNfOPe1OBPZIcv/W932SPGQSNX0N+ABwwtDyU2jTu5NsDvwdcNEEfd2UZK3xGiS5G/DgqjoZeAuwIbD+JOqUJEmSpClzpHn1cguwB3Bokg3pPr8PAReMs82pwCXAT+mmUJ89qlFV/TTJ24HvtWB6E915yZeOV1BV/RV4P0Cb1j3m48BhSRbRjXbvXVU3DrUZdjhwXpKzgQN62swAjm7HH+DQqrp6vE4lSZIkaVmlqqa7Bmmleu0B76vvLt1musuQJGmFWnzQnOkuQZLWJL2je07PliRJkiSph9OzNa4kp9NdtXvQS6pq0XTUI0mSJEl3JkOzxlVVj5nuGiRJkiRpujg9W5IkSZKkHoZmSZIkSZJ6GJolSZIkSephaJYkSZIkqYehWZIkSZKkHoZmSZIkSZJ6GJolSZIkSephaJYkSZIkqYehWZIkSZKkHoZmSZIkSZJ6zJzuAqSVbetNN+SwfedMdxmSJEmSVkOONEuSJEmS1MPQLEmSJElSD0OzJEmSJEk9DM2SJEmSJPUwNEuSJEmS1MPQLEmSJElSD0OzJEmSJEk9DM2SJEmSJPUwNEuSJEmS1GPmdBcgrWyLLlvCrP2Pm+4yJN2FLT5oznSXIEmSlpEjzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST1Wm9Cc5JpJtNkvyXp3Qi2zk/zjBG0ekOTbSc5N8tMk31mB+//xCuhjtyRLkixsj/9NsmOSQ9v6uUn+ffmrXeb6npNki4HX70rylOmqR5IkSdJd08zpLmAF2w84GrhushskmVFVS6e4n9nAjsB4QfhdwPer6sNtP9tMoaYAqapbRq2vqsdOodbxnFJVzxxatmB5O52o/kl6DvBt4KcAVfWO5a1LkiRJkqZqtRlpHtNGSOcn+XKSC5Mck84bgE2Ak5Oc3No+LclPkpyd5EtJ1m/LFyd5f5KzgRe01we2douSPKq1u0eSI5KckeScJM9OsjZdIN6zjdDu2VPqxsBvx15U1XkDx/DmJGcmOS/JgW3ZrCQXJfkMcD7wn0kOHthm7yQfbc+vGVj+llbzuUkOass2S3J8krOSnDJ2PJN8b789sGjb9v79Iskrp1j/g4f6fnJ7Dxe193Sdgc/iA235GUkenuSxwLOAg9t7vFmSeUn2aNvslOTH7ZjPSLLBiGN5VZIFSRYsvW7JZA5fkiRJku5gtQvNzXZ0o8pbAA8DHldVhwK/A3avqt2T3Bd4O/CUqtqebgT1Xwf6uLKqtq+qY9vrK1q7w4CxackHACdV1c7A7sDBwFrAO4AvVNXsqvpCT40fA/4nyclJDkiyCXRBHngEsDPdiPUOSZ7QtnkE8PGq2hL4OPDcgf72BI4deE2SZwDPBh5TVdsCH2irDgdeX1U7tGP5eE+Nuw5Mzz5gxPptgCcBuwDvSLLJZOuvqksH6lwXmAfsWVVb081weO3Afpa05R8FPlRVPwa+Cby5vccXD/S1NvAF4I3tmJ8CXD9ceFUdXlU7VtWOM9bbsOfwJUmSJGl8q+v07DOq6rcASRYCs4AfDbX5e7pQfWo3W5i1gZ8MrB8Ou19tP88CnteePw141sC5vesCfzeZAqvqhCQPA54OPAM4J8lWrc+nAee0puvThc1fA5dW1Wlt+8uT/CrJ3wO/AB4FnDq0m6cAR1bVdW2bq9po+mOBL7XjBlinp8zbTc9OstvQ+m9U1fXA9W30fmfg8ZOpf8gjgUuq6uft9VHA64APtdefH/h5SE+tg339vqrOBKiqv0zQXpIkSZKW2eoamm8ceL6U0ccRunOK9+rp49qePgf7C/D8qrrodh0nj5lMkVV1FfA54HNt2vMTWp/vq6pPDvU5a0RNxwL/B7gQ+FpV1SR2ezfg6qqaPZkaJzC8v2Jq9S/LfiZzjJIkSZJ0p1hdp2f3+Sswdn7racDjkjwcbj0/efMp9ncC8Pp2YSuSbDdiPyMleVLalbzbObeb0Y3GngC8fOD86k2T3L+nm6/RTb/ei6Gp2c33gX0G9nOfNvJ6SZIXtGVJsu1kD3jIs5Osm2QjYDfgzCnWP+YiYNbYZwG8BPjBwPo9B36OzQboe48vAjZOslPb/wZJVtcvfyRJkiSt4ta00Hw4cHySk6vqcmBv4PNJzqMLY5O6INaAd9Odw3xekgvaa4CTgS0muBDYDsCCgX1/uqrOrKrv0Y0+/yTJIuDL9ATwqvoz8DPgIVV1xoj1x9Od+7ugTVMfm0b+YuAVSc4FLqAL3iR5VpJ3TeH4z2vHehrw7qr63VTqH6jzBmAfuinji4BbgE8MNLl3e5/eCLypLTsWeHO7eNhmA339jS5cf6Qd3/fpps1LkiRJ0gqXyc34lVaOJIuBHavqipW1j9ce8L767tJJ3/FLkla4xQfNme4SJEnS+NK3Yk0baZYkSZIkaYXxXNDllGQfumnFg06tqtdNRz2rm6qaNd01SJIkSVIfQ/NyqqojgSOnuw5JkiRJ0orn9GxJkiRJknoYmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQehmZJkiRJknoYmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQehmZJkiRJknoYmiVJkiRJ6jFzuguQVratN92Qw/adM91lSJIkSVoNOdIsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVKPmdNdgLSyLbpsCbP2P266y5C0Clt80JzpLkGSJK2iHGmWJEmSJKmHoVmSJEmSpB6GZkmSJEmSehiaJUmSJEnqYWiWJEmSJKmHoVmSJEmSpB6GZkmSJEmSehiaJUmSJEnqYWiWJEmSJKmHoVmSJEmSpB6GZkmSJEmSehiaJUmSJEnqYWiWJEmSJKmHoVmSJEmSpB6GZkmSJEmSehiaJUmSJEnqYWiWJEmSJKmHoXklSfKgJN9I8oskFyf5cJK1J9hmcZL7tuc/XsH1/H2S05MsTPKzJHNXZP8j9rdjkkPb892SPHYK285PsmCor/kroUxJkiRJGpeheSVIEuCrwNer6hHA5sD6wHsn20dVTTpkTtJRwKuqajawFfDFFdz/rZLMrKoFVfWGtmg3YKrHc/8kz1ixlUmSJEnS1BiaV44nATdU1ZEAVbUUeBPw8iT7JvlqkuPbKPQHRnWQ5Jr2c7c28vrlJBcmOaaFcpLskOQHSc5KckKSjcep6f7A78fqqaqftj7ukeSIJGckOSfJs9vyGUn+K8n5Sc5L8vq2fHA0/NYR4CRzk3w2yanAZ1vd304yC3gN8KY2yr1rkkuSrNW2u+fg6wEHAweMeF/WTXJkkkWt3t173r9XJVmQZMHS65aM87ZIkiRJUj9D88qxJXDW4IKq+gvwa2AmMBvYE9ga2DPJgyfobztgP2AL4GHA41rI/AiwR1XtABzB+CPZhwAXJflaklcnWbctPwA4qap2BnYHDk5yD+BVwCxgdlVtAxwziePeAnhKVe01cNyLgU8Ah1TV7Ko6BZgPzGlNXgh8tapuGurrJ8DfRoTi13Xd1tbAXsBRA8dyq6o6vKp2rKodZ6y34SRKlyRJkqQ7MjRPjxOraklV3QD8FHjIBO3PqKrfVtUtwEK6MPtIumnW30+yEHg78KC+DqrqXcCOwPeAFwHHt1VPA/ZvfcwH1gX+DngK8Mmqurltf9UkjuubVXX9JNp9GtinPd8HOLKn3XvojmvQ44GjW00XApfSTX+XJEmSpBVu5nQXsIb6KbDH4IIk96QLozcDNw6sWsrEn8Oo9gEuqKpdJltUVV0MHJbkU8DlSTZq/Ty/qi4aqrevm5u57cuW4RHeaydZx6lJZiXZDZhRVef3tDspyXuAv59Mv5IkSZK0ojnSvHKcCKyX5KXQnR8MfBCYB1y3gvZxEXC/JLu0fayVZMu+xknm5LYk/Ai68H01cALw+oHzpLdrbb4PvDrJzLb8Pm35YmCH9vz5k6z1r8AGQ8s+A3yO/lHmMe8B/mPg9SnAi1tNm9N9EXHRiO0kSZIkabkZmleCqirgucALkvwC+DlwA/C2FbiPv9GNZr8/ybl007bHu0L1S+jOaV4IfBZ4cbtA2buBtYDzklzQXkM3hfrXbfm5dFO6AQ4EPtxuCbV0kuV+C3ju2IXA2rJjgHsDnwdI8p0km4w4zu8Alw8s+jhwtySLgC8Ae1fVjcPbSZIkSdKKkC7fSXeuJHsAz66ql6zsfb32gPfVd5dus7J3I2k1tvigORM3kiRJa7Le81M9p1l3uiQfAZ4B/ON01yJJkiRJ4zE0r2GSfAx43NDiD4/dM3pVUFWvn+4aJEmSJGkyDM1rmKp63XTXIEmSJElrCi8EJkmSJElSD0OzJEmSJEk9DM2SJEmSJPUwNEuSJEmS1MPQLEmSJElSD0OzJEmSJEk9DM2SJEmSJPUwNEuSJEmS1MPQLEmSJElSj5nTXYC0sm296YYctu+c6S5DkiRJ0mrIkWZJkiRJknoYmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQehmZJkiRJknoYmiVJkiRJ6mFoliRJkiSph6FZkiRJkqQehmZJkiRJknrMnO4CpJVt0WVLmLX/cdNdhu4iFh80Z7pLkCRJ0grkSLMkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ7MkSZIkST0MzZIkSZIk9TA0S5IkSZLUw9AsSZIkSVIPQ/OdIMkDkxyb5OIkZyX5TpLNl7Gv/ZKst4LqmpXkRSOWb5RkYXv8IcllA6/XXhH7liRJkqTVgaF5JUsS4GvA/KrarKp2AN4KPGAZu9wPGBmak8yYYl+zgDuE5qq6sqpmV9Vs4BPAIWOvq+pvUy14ZUsyc7prkCRJkrRmMjSvfLsDN1XVJ8YWVNW5VXVKOgcnOT/JoiR7AiTZLcn8JF9OcmGSY1rbNwCbACcnObm1vSbJB5OcC+yS5B1J/n979x5lZ1Wnefz7mCAiYFoFbRuVIIjIHQwMiigIXjCKiDhe8ILtQIsKgmJDD64RZqSJskZGREHECyINNip2WloBuQgIwYRbArS2iBlFbZFRIoiAlr/54+zqHNP1Vp2EpE6q/H7WYtU5+93vfn9v1V4JT+193ixsY57ZQjtJtkjyrSS3JLkxyebAPGCPtoJ81Hg3keRJSW5or3dIUkme3t7/MMlj28r15UkWJ7ls9HjfGI9K8oMkG/e9vyPJxu2/r7TaFybZvfXZNcl1SW5Kcm2SZ7X2g5PMT3I5cNkj/ilJkiRJ0hgMzWvetsANHccOAHYEdgD2AU5O8pR2bCd6q8pbA88Adq+qU4GfAXtV1V6t3/rA9VW1Q1VdA5xWVbtU1bbAesArWr9zgU9U1Q7A84CfA8cCV7cV5FPGu4mquht4TJLHAXsAi+gF7k2Bu6vqAeDjwNlVtX273qkrjPFH4IvAQa1pH+CWqvol8DF6K9q7AK8Bzmp9vgfsUVU7Af8D+Pu+IXcGDqyqF65Yb5JDkyxKsmjkgWXj3ZokSZIkdTI0D9fzgfOqaqSqfgF8G9ilHftuVd3VgubN9LZSj2UE+Erf+72SXJ9kCfAiYJskGwKbVNWFAFX1YAu5K+taYHfgBfTC6wvoBeir2/HnAv/QXp/T7m9FnwXe0l7/NfC59nof4LQkNwPzgccl2QCYBVyQ5FbgFGCbvrEurapfjVVoVZ1ZVXOqas6Mx85a6RuVJEmSJAA/C7rm3QYcuArnPdT3eoTun9WDVTUCkOQxwCeBOVX1kyTHA49ZhWt3uYpeSN4U+CfgGKCAiwYdoNX1iyQvAnZl+arzo4DdqurB/v5JTgOuqKpXJ5kNXNl3+LerdhuSJEmSNBhXmte8y4F1kxw62pBk+ySjK7SvSzKjfc73BcB3JxjvPmDDjmOjAfmetkp7IEBV3QfclWT/dv112xO4xxtrLFcDbwJ+0FbAfwW8HLimHb8WeH17fRDLV6BXdBa9bdoXjAZ+4BLg8NEOSXZsL2cBP22vD16JWiVJkiTpETM0r2FVVcCrgX3aA7NuA04C/p3eU7UXA7fQC9d/W1X/PsGQZwLfHH0Q2ArXuhf4NHArcDGwsO/wm4EjkiymF27/sl17pD0c7Kgkf5XkX8a5l6VA6K04Qy8s31tVv27vDwfe1q7xZuA9HUPNBzZg+dZsgCOAOe0hYrcD72jtHwFOSnIT7oyQJEmSNMnSy3TS5Ekyh95Dv/aYjOsddtxJ9Y2R7SfjUhJL580ddgmSJElaeek64MqdJlWSY4HDWP5ZZkmSJElaa7k9W5OqquZV1abtn8eSJEmSpLWaoVmSJEmSpA6GZkmSJEmSOhiaJUmSJEnqYGiWJEmSJKmDoVmSJEmSpA6GZkmSJEmSOhiaJUmSJEnqYGiWJEmSJKmDoVmSJEmSpA6GZkmSJEmSOhiaJUmSJEnqYGiWJEmSJKnDzGEXIK1p220yi9PfOXfYZUiSJEmaglxpliRJkiSpg6FZkiRJkqQOhmZJkiRJkjoYmiVJkiRJ6mBoliRJkiSpg6FZkiRJkqQOhmZJkiRJkjoYmiVJkiRJ6mBoliRJkiSpw8xhFyCtaUt+uozZx1407DI0iZbOmzvsEiRJkjRNuNIsSZIkSVIHQ7MkSZIkSR0MzZIkSZIkdTA0S5IkSZLUwdAsSZIkSVIHQ7MkSZIkSR0MzZIkSZIkdTA0S5IkSZLUwdAsSZIkSVIHQ7MkSZIkSR0MzZIkSZIkdTA0S5IkSZLUwdAsSZIkSVIHQ7MkSZIkSR0MzZIkSZIkdTA0S5IkSZLUwdC8lkoykuTmJLcluSXJ+5IM7eeVZGmSjVbx3P2TbL0S/WcnqSSH97WdluTgVbm+JEmSJK0qQ/Pa63dVtWNVbQO8GNgX+OCQa1pV+wMDh+bmbuA9SR69BuqRJEmSpIEYmqeAqrobOBR4d3pmJDk5ycIki5P8DUCSPZNcleSiJN9Pcsbo6nSSlyS5LsmNSS5IskFrX5rkhNa+JMlWrf2JSS5pK91nARmtJ8mbkny3rYR/KsmM1n5/khPbyviCJE9O8jxgP+Dk1n/zJEckub3Vfn7Hbf8SuAx464oHkuzYxl+c5MIkj19d32tJkiRJ6mdoniKq6k5gBvAk4O3AsqraBdgFOCTJZq3rrsDh9FZ2NwcOaNuqPwDsU1U7A4uA9/YNf09rPx04urV9ELimrXRfCDwdIMmzgdcBu1fVjsAIcFA7Z31gQVXtAFwFHFJV1wLzgfe3lfMfAscCO1XV9sA7xrntDwNHj4byPl8AjmnnL2GMFfgkhyZZlGTRyAPLxrmEJEmSJHUzNE9NLwHekuRm4HrgicAz27HvVtWdVTUCnAc8H9iNXoj+TjvnrcCmfeN9tX29AZjdXr8A+CJAVV0E/Lq17w08B1jYxtobeEY79jDw9THGWtFi4NwkbwL+0HWT7RcF1wNvHG1LMgv4i6r6dms6u9W64rlnVtWcqpoz47Gzui4hSZIkSeOaOewCNJgkz6C3qns3va3Sh1fVxSv02ROoFU6t1v/SqnpDx/APta8jTDwnApxdVX83xrHfV9Xo9ccbay69oPtK4Lgk21VVV3j+e+DLwLc7jkuSJEnSGuNK8xSQZGPgDOC0FkovBg5Lsk47vmWS9Vv3XZNs1j7L/DrgGmABsHuSLVr/9ZNsOcFlr6Kt8CbZFxj93PBlwIFJntSOPSHJpmMP8R/uAzZs/R8FPK2qrgCOAWYBG3SdWFXfA26nF7CpqmXAr5Ps0bq8GQO1JEmSpDXElea113pt+/M69LYwnwN8tB07i97W5xuThN5Ds/ZvxxYCpwFbAFcAF1bVH9s/13ReknVbvw8A/zbO9U9o/W8DrgV+DFBVtyf5AHBJC8C/B94F/N9xxjof+HSSI4DXA59p26wDnFpV9yaZA7yjqv7bGOefCNzU9/6twBlJHgvcCbxtnGtLkiRJ0irL8t20mura9uyjq+oVw65lbXLYcSfVN0a2H3YZmkRL580ddgmSJEmaWtJ1wO3ZkiRJkiR1cHv2NFJVVwJXDrkMSZIkSZo2XGmWJEmSJKmDoVmSJEmSpA6GZkmSJEmSOhiaJUmSJEnqYGiWJEmSJKmDoVmSJEmSpA6GZkmSJEmSOhiaJUmSJEnqYGiWJEmSJKmDoVmSJEmSpA6GZkmSJEmSOswcdgHSmrbdJrM4/Z1zh12GJEmSpCnIlWZJkiRJkjoYmiVJkiRJ6mBoliRJkiSpg6FZkiRJkqQOhmZJkiRJkjoYmiVJkiRJ6mBoliRJkiSpg6FZkiRJkqQOhmZJkiRJkjrMHHYB0pq25KfLmH3sRcMuQ6vB0nlzh12CJEmS/sy40ixJkiRJUgdDsyRJkiRJHQzNkiRJkiR1MDRLkiRJktTB0CxJkiRJUgdDsyRJkiRJHQzNkiRJkiR1MDRLkiRJktTB0CxJkiRJUgdDsyRJkiRJHQzNkiRJkiR1MDRLkiRJktTB0CxJkiRJUgdDsyRJkiRJHQzNkiRJkiR1MDRLkiRJktTB0LwWSTKS5OYktyW5Jcn7kgztZ5RkaZKNVvHc/ZNsPWDfxyT5XpLt+tren+RTq3JtSZIkSVpdDM1rl99V1Y5VxDDtVwAAERxJREFUtQ3wYmBf4INDrmlV7Q8MFJqr6kHgSOCT6dkEeAdw7BqsT5IkSZImZGheS1XV3cChwLtbkJyR5OQkC5MsTvI3AEn2THJVkouSfD/JGaOr00lekuS6JDcmuSDJBq19aZITWvuSJFu19icmuaStdJ8FZLSeJG9K8t22Ev6pJDNa+/1JTmwr4wuSPDnJ84D9gJNb/82THJHk9lb7+WPc7zeBnwNvAU4BjgdmJvlKu+eFSXZv13xhG/fmJDcl2XAN/RgkSZIk/ZkzNK/FqupOYAbwJODtwLKq2gXYBTgkyWat667A4fRWdjcHDmjbqj8A7FNVOwOLgPf2DX9Paz8dOLq1fRC4pq10Xwg8HSDJs4HXAbtX1Y7ACHBQO2d9YEFV7QBcBRxSVdcC84H3t5XzH9JbNd6pqrant4o8liOBE4GNq+oc4GPAKe2eXwOc1fodDbyr1bIH8LsVB0pyaJJFSRaNPLCs61ssSZIkSeOaOewCNLCXANsnObC9nwU8E3gY+G4L2CQ5D3g+8CC9EP2dJACPBq7rG++r7esNwAHt9QtGX1fVRUl+3dr3Bp4DLGxjrQfc3Y49DHy9b6wXd9S/GDg3ydeAr43Voap+luTyvvH2AbZu1wR4XFst/w7w0STnAl+tqrvGGOtM4EyAw447qRjpqEqSJEmSxmFoXosleQa9Vd276W2VPryqLl6hz55ArXBqtf6XVtUbOoZ/qH0dYeJ5EODsqvq7MY79vqpGrz/eWHPphfJXAscl2a6q/jBGvz+2/6C3E2K39pnnfvOSXAS8nN4vBV5aVd+b4B4kSZIkaaW5PXstlWRj4AzgtBZKLwYOS7JOO75lkvVb912TbNY+y/w64BpgAbB7ki1a//WTbDnBZa8C3tj67ws8vrVfBhyY5Ent2BOSbDrBWPcBG7b+jwKeVlVXAMfQWyXfYIBvwyX0tp3Txtmxfd28qpZU1YeBhcBWA4wlSZIkSSvN0Lx2WW/0n5wCvkUvNJ7Qjp0F3A7cmORW4FMsX9VdCJwG/CvwI+DCqvolcDBwXpLF9LZmTxQuTwBe0K5/APBjgKq6nd7noy9pY10KPGWCsc4H3p/kJnrbyL+YZAlwE3BqVd2bZE574FiXI4A57eFht7P8s9BHJrm11fJ74BsT1CJJkiRJqyTLd9ZqKmrbs4+uqlcMu5a11WHHnVTfGNl+2GVoNVg6b+6wS5AkSdL0lK4DrjRLkiRJktTBB4FNcVV1JXDlkMuQJEmSpGnJlWZJkiRJkjoYmiVJkiRJ6mBoliRJkiSpg6FZkiRJkqQOhmZJkiRJkjoYmiVJkiRJ6mBoliRJkiSpg6FZkiRJkqQOhmZJkiRJkjoYmiVJkiRJ6mBoliRJkiSpg6FZkiRJkqQOM4ddgLSmbbfJLE5/59xhlyFJkiRpCnKlWZIkSZKkDoZmSZIkSZI6GJolSZIkSepgaJYkSZIkqYOhWZIkSZKkDoZmSZIkSZI6GJolSZIkSepgaJYkSZIkqYOhWZIkSZKkDjOHXYC0pi356TJmH3vRsMuYNpbOmzvsEiRJkqRJ40qzJEmSJEkdDM2SJEmSJHUwNEuSJEmS1MHQLEmSJElSB0OzJEmSJEkdDM2SJEmSJHUwNEuSJEmS1MHQLEmSJElSB0OzJEmSJEkdDM2SJEmSJHUwNEuSJEmS1MHQLEmSJElSB0OzJEmSJEkdDM2SJEmSJHUwNEuSJEmS1MHQLEmSJElShwlDc5KRJDcnuTXJBUkeOxmFDSLJnkmeN0Gf45NUki362o5sbXMmOPfI/vtNcv8jrHfM85MsTbKkfZ9vTnJqa/98kgNX4Tqzk7zxkdQ6xpjXPsLzZye5dTXUsVrGkSRJkqRBDLLS/Luq2rGqtgUeBt6xhmtaGXsC44bmZgnw+r73rwVuG+C8I4HJ+iXBXu37vGNVHfEIx5oNjBmak8xclQGrapDvsyRJkiRNKyu7PftqYIskr0xyfZKbknwryZOTPCrJD5JsDNDe35Fk47ZienqSBUnubCvEn03yr0k+Pzp4kpckuS7JjW1Ve4PWvjTJCa19SZKtksymF+CPaquze4xT99eAV7WxNgeWAfeMd90kRwB/BVyR5Iq+vicmuaXdy5Nb2+wklydZnOSyJE9v7Zu1cZck+dBKfq//RJLnJPl2khuSXJzkKa19i/YzuKXVvzkwD9ijfV+OSnJwkvlJLgcuS/KEJF9r9S5Isn0b6/j2c7my/ZyO6Lv+/X2vj2n3dEuSeWPU+uQkF7bjt/TtBpiR5NNJbktySZL1Rn8mSb7Z7u3qJFtNMM7odZ7R5uAuY9RwaJJFSRaNPLDskXzrJUmSJP0ZGzg0txXKfemt2l4D7FZVOwHnA39bVX8Evggc1E7ZB7ilqn7Z3j8eeC5wFDAfOAXYBtguyY5JNgI+AOxTVTsDi4D39pVwT2s/HTi6qpYCZwCntNXZq8cp/zfAT5JsS2/F+Ut99zXmdavqVOBn9FaA92rd1wcWVNUOwFXAIa3948DZVbU9cC5wamv/GHB6VW0H/Hyc+qAXzke3Zx/VfyDJOu0aB1bVc4DPAie2w+cCn2g1Pa9d51jg6vZ9OaX127md/0LgBOCmVu9/B77Qd7mtgJcCuwIfbNfur2Vfer+A+C/tmh8Z415OBb7dju/M8lX9Z7ZatwHuBV7T2s8EDm/3djTwyQnGIcmzgK8AB1fVwhULqKozq2pOVc2Z8dhZY5QoSZIkSRMbZKvueklubq+vBj4DPAv4UlvtfDTwo3b8s8A/Af8H+Gvgc33j/HNVVZIlwC+qaglAktvobSd+KrA18J0ktHGv6zv/q+3rDcABK3GPo86nF5hfCuwNvK217zbBdfs9DHy9r44Xt9fP7avpHJYHyd1ZHgzPAT48Tn17VdU9HceeBWwLXNpqnAH8PMmGwCZVdSFAVT0I0Pqs6NKq+lV7/fzRuqrq8iRPTPK4duyiqnoIeCjJ3cCTgbv6xtkH+FxVPdDO/xX/2YuAt7TjI8CyJI8HflRVo3PpBmB2203wPOCCvrrXnWCcjenNswOq6vaO75kkSZIkPWKDhObfVdWO/Q1JPg58tKrmJ9kTOB6gqn6S5BdJXkRvpfKgvtMeal//2Pd69P1MYIResHtDRx2j54wMWPeKvg6cDCyqqt/0BbRMcN1+v6+qWsk6auIuEwpwW1U9908ae6F5UL8dsF//z2ZVv9eDjr0evd0O9644xyawDPgxvfBvaJYkSZK0xqzqPzk1C/hpe/3WFY6dRW+b9gVtdXBQC4Dd055ynWT9JFtOcM59wEDBsa2MHsPybc2DXHfQ8a9l+YPGDqK3Ig/wnRXaV9X3gY2TPLfVuE6SbarqPuCuJPu39nXTe9r3RHVfPVpP+6XHPVX1mwFruRR4W7sOSZ4wRp/LgMPa8RlJOvdHt+v+KMlrW/8k2WGCcR4GXg28Jav5KeGSJEmS1G9VQ/Px9LbT3kDfA7Wa+cAG/OnW7Am1zz4fDJyXZDG9LdJbTXDaPwOvzsQPAhu9xvlVdeNKXPdM4JvpexBYh8PpBcnFwJuB97T29wDvalvSN+k/oW/L+6j+zzT3f8aYqnoYOBD4cJJbgJtZ/tTwNwNHtGtfC/wlsBgYaQ/P+pPPRzfHA89p58zjP//io1NVfZPez3hRu4ejx+j2HmCvdt830Nv+Pp6DgLe3e7uN9tC28capqt8Cr6D3ILj9Bq1fkiRJklZGlu82Xk0D9v7t41OqasIQK02Gw447qb4xsv2wy5g2ls6bO+wSJEmSpNVtzAdDwer9vCpJjqW3nfaRbEWWJEmSJGmtsFpDc1XNo7fdd9IlOQ547QrNF1TVip9hliRJkiRpIKs1NA9TC8cGZEmSJEnSarOqDwKTJEmSJGnaMzRLkiRJktTB0CxJkiRJUgdDsyRJkiRJHQzNkiRJkiR1MDRLkiRJktTB0CxJkiRJUgdDsyRJkiRJHQzNkiRJkiR1MDRLkiRJktRh5rALkNa07TaZxenvnDvsMiRJkiRNQa40S5IkSZLUwdAsSZIkSVIHQ7MkSZIkSR0MzZIkSZIkdTA0S5IkSZLUwdAsSZIkSVIHQ7MkSZIkSR0MzZIkSZIkdTA0S5IkSZLUwdAsSZIkSVIHQ7MkSZIkSR0MzZIkSZIkdTA0S5IkSZLUwdAsSZIkSVIHQ7MkSZIkSR0MzZIkSZIkdTA0S5IkSZLUwdAsSZIkSVIHQ7MkSZIkSR0MzZIkSZIkdTA0S5IkSZLUwdAsSZIkSVIHQ7MkSZIkSR0MzZIkSZIkdTA0S5IkSZLUwdAsSZIkSVIHQ7MkSZIkSR0MzZIkSZIkdTA0S5IkSZLUIVU17BqkNeqYY465b5111vn+sOvQ9HH//fdvtMEGG9wz7Do0fTintDo5n7S6Oae0uq2lc+qeD33oQy8b64ChWdNekkVVNWfYdWj6cE5pdXNOaXVyPml1c05pdZtqc8rt2ZIkSZIkdTA0S5IkSZLUwdCsPwdnDrsATTvOKa1uzimtTs4nrW7OKa1uU2pO+ZlmSZIkSZI6uNIsSZIkSVIHQ7MkSZIkSR0MzZo2krwsyfeT3JHk2DGOr5vkS+349UlmT36VmioGmE/vTXJ7ksVJLkuy6TDq1NQx0Zzq6/eaJJVkyvxTHBqOQeZUkv/a/qy6Lck/THaNmloG+Lvv6UmuSHJT+/vv5cOoU1NDks8muTvJrR3Hk+TUNt8WJ9l5smsclKFZ00KSGcAngH2BrYE3JNl6hW5vB35dVVsApwAfntwqNVUMOJ9uAuZU1fbAl4GPTG6VmkoGnFMk2RB4D3D95FaoqWaQOZXkmcDfAbtX1TbAkZNeqKaMAf+c+gDwj1W1E/B64JOTW6WmmM8DLxvn+L7AM9t/hwKnT0JNq8TQrOliV+COqrqzqh4GzgdetUKfVwFnt9dfBvZOkkmsUVPHhPOpqq6oqgfa2wXAUye5Rk0tg/wZBfC/6P1C78HJLE5T0iBz6hDgE1X1a4CqunuSa9TUMsicKuBx7fUs4GeTWJ+mmKq6CvjVOF1eBXyhehYAf5HkKZNT3coxNGu62AT4Sd/7u1rbmH2q6g/AMuCJk1KdpppB5lO/twPfWKMVaaqbcE61bWlPq6qLJrMwTVmD/Dm1JbBlku8kWZBkvBUfaZA5dTzwpiR3Af8CHD45pWmaWtn/3xqamcMuQJKmsiRvAuYALxx2LZq6kjwK+Chw8JBL0fQyk962xz3p7Ya5Ksl2VXXvUKvSVPYG4PNV9b+TPBc4J8m2VfXHYRcmrUmuNGu6+CnwtL73T21tY/ZJMpPetqL/NynVaaoZZD6RZB/gOGC/qnpokmrT1DTRnNoQ2Ba4MslSYDdgvg8D0zgG+XPqLmB+Vf2+qn4E/Bu9EC2NZZA59XbgHwGq6jrgMcBGk1KdpqOB/n9rbWBo1nSxEHhmks2SPJrewynmr9BnPvDW9vpA4PKqqkmsUVPHhPMpyU7Ap+gFZj8nqImMO6eqallVbVRVs6tqNr3Pye9XVYuGU66mgEH+3vsavVVmkmxEb7v2nZNZpKaUQebUj4G9AZI8m15o/uWkVqnpZD7wlvYU7d2AZVX182EXNRa3Z2taqKo/JHk3cDEwA/hsVd2W5H8Ci6pqPvAZetuI7qD3UILXD69irc0GnE8nAxsAF7Tnyf24qvYbWtFaqw04p6SBDTinLgZekuR2YAR4f1W5w0pjGnBOvQ/4dJKj6D0U7GAXINQlyXn0fnG3Ufsc/AeBdQCq6gx6n4t/OXAH8ADwtuFUOrE4zyVJkiRJGpvbsyVJkiRJ6mBoliRJkiSpg6FZkiRJkqQOhmZJkiRJkjoYmiVJkiRJ6mBoliRJkiSpg6FZkiRJkqQO/x/WCZKmnh796gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1008x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mzfOwj0vwmW4",
        "outputId": "bd995b78-6d06-490d-f0e5-55f0152b0520",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "#Download the mojo file for deployment\n",
        "aml.leader.download_mojo(path = \"./\")"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/GLM_1_AutoML_20201020_090123.zip'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bt2A7D7pKDCj"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}