testing-fast-ml-built-in-functions.ipynb

{"cells":[{"source":"<a href=\"https://www.kaggle.com/code/imenbakir/testing-fast-ml-built-in-functions?scriptVersionId=99468028\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown","outputs":[],"execution_count":0},{"cell_type":"code","execution_count":1,"id":"ddcaf122","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2022-06-27T19:52:30.723335Z","iopub.status.busy":"2022-06-27T19:52:30.722644Z","iopub.status.idle":"2022-06-27T19:52:30.743069Z","shell.execute_reply":"2022-06-27T19:52:30.742245Z"},"papermill":{"duration":0.030688,"end_time":"2022-06-27T19:52:30.745474","exception":false,"start_time":"2022-06-27T19:52:30.714786","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["/kaggle/input/student-performance-data/student_data.csv\n"]}],"source":["# This Python 3 environment comes with many helpful analytics libraries installed\n","# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n","# For example, here's several helpful packages to load\n","\n","import numpy as np # linear algebra\n","import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n","\n","# Input data files are available in the read-only \"../input/\" directory\n","# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n","\n","import os\n","for dirname, _, filenames in os.walk('/kaggle/input'):\n","    for filename in filenames:\n","        print(os.path.join(dirname, filename))\n","\n","# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n","# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"]},{"cell_type":"code","execution_count":2,"id":"532a7b13","metadata":{"execution":{"iopub.execute_input":"2022-06-27T19:52:30.756224Z","iopub.status.busy":"2022-06-27T19:52:30.755214Z","iopub.status.idle":"2022-06-27T19:52:46.051601Z","shell.execute_reply":"2022-06-27T19:52:46.050215Z"},"papermill":{"duration":15.304509,"end_time":"2022-06-27T19:52:46.054402","exception":false,"start_time":"2022-06-27T19:52:30.749893","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Collecting fast_ml\r\n","  Downloading fast_ml-3.68-py3-none-any.whl (42 kB)\r\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.1/42.1 kB\u001b[0m \u001b[31m181.2 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n","\u001b[?25hInstalling collected packages: fast_ml\r\n","Successfully installed fast_ml-3.68\r\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n","\u001b[0m"]},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>data_type</th>\n","      <th>data_type_grp</th>\n","      <th>num_unique_values</th>\n","      <th>sample_unique_values</th>\n","      <th>num_missing</th>\n","      <th>perc_missing</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>school</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[GP, MS]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>sex</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[F, M]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>age</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>8</td>\n","      <td>[18, 17, 15, 16, 19, 22, 20, 21]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>address</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[U, R]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>famsize</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[GT3, LE3]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>Pstatus</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[A, T]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>Medu</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>5</td>\n","      <td>[4, 1, 3, 2, 0]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>Fedu</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>5</td>\n","      <td>[4, 1, 2, 3, 0]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>Mjob</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>5</td>\n","      <td>[at_home, health, other, services, teacher]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>Fjob</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>5</td>\n","      <td>[teacher, other, services, health, at_home]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>reason</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>4</td>\n","      <td>[course, other, home, reputation]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>guardian</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>3</td>\n","      <td>[mother, father, other]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>traveltime</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>4</td>\n","      <td>[2, 1, 3, 4]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>studytime</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>4</td>\n","      <td>[2, 3, 1, 4]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>failures</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>4</td>\n","      <td>[0, 3, 2, 1]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>schoolsup</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[yes, no]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>famsup</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[no, yes]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>paid</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[no, yes]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>activities</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[no, yes]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>nursery</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[yes, no]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>higher</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[yes, no]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>internet</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[no, yes]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>romantic</th>\n","      <td>object</td>\n","      <td>Categorical</td>\n","      <td>2</td>\n","      <td>[no, yes]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>famrel</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>5</td>\n","      <td>[4, 5, 3, 1, 2]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>freetime</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>5</td>\n","      <td>[3, 2, 4, 1, 5]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>goout</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>5</td>\n","      <td>[4, 3, 2, 1, 5]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>Dalc</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>5</td>\n","      <td>[1, 2, 5, 3, 4]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>Walc</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>5</td>\n","      <td>[1, 3, 2, 4, 5]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>health</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>5</td>\n","      <td>[3, 5, 1, 2, 4]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>absences</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>34</td>\n","      <td>[6, 4, 10, 2, 0, 16, 14, 7, 8, 25]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>G1</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>17</td>\n","      <td>[5, 7, 15, 6, 12, 16, 14, 10, 13, 8]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>G2</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>17</td>\n","      <td>[6, 5, 8, 14, 10, 15, 12, 18, 16, 13]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>G3</th>\n","      <td>int64</td>\n","      <td>Numerical</td>\n","      <td>18</td>\n","      <td>[6, 10, 15, 11, 19, 9, 12, 14, 16, 5]</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["           data_type data_type_grp num_unique_values  \\\n","school        object   Categorical                 2   \n","sex           object   Categorical                 2   \n","age            int64     Numerical                 8   \n","address       object   Categorical                 2   \n","famsize       object   Categorical                 2   \n","Pstatus       object   Categorical                 2   \n","Medu           int64     Numerical                 5   \n","Fedu           int64     Numerical                 5   \n","Mjob          object   Categorical                 5   \n","Fjob          object   Categorical                 5   \n","reason        object   Categorical                 4   \n","guardian      object   Categorical                 3   \n","traveltime     int64     Numerical                 4   \n","studytime      int64     Numerical                 4   \n","failures       int64     Numerical                 4   \n","schoolsup     object   Categorical                 2   \n","famsup        object   Categorical                 2   \n","paid          object   Categorical                 2   \n","activities    object   Categorical                 2   \n","nursery       object   Categorical                 2   \n","higher        object   Categorical                 2   \n","internet      object   Categorical                 2   \n","romantic      object   Categorical                 2   \n","famrel         int64     Numerical                 5   \n","freetime       int64     Numerical                 5   \n","goout          int64     Numerical                 5   \n","Dalc           int64     Numerical                 5   \n","Walc           int64     Numerical                 5   \n","health         int64     Numerical                 5   \n","absences       int64     Numerical                34   \n","G1             int64     Numerical                17   \n","G2             int64     Numerical                17   \n","G3             int64     Numerical                18   \n","\n","                                   sample_unique_values num_missing  \\\n","school                                         [GP, MS]           0   \n","sex                                              [F, M]           0   \n","age                    [18, 17, 15, 16, 19, 22, 20, 21]           0   \n","address                                          [U, R]           0   \n","famsize                                      [GT3, LE3]           0   \n","Pstatus                                          [A, T]           0   \n","Medu                                    [4, 1, 3, 2, 0]           0   \n","Fedu                                    [4, 1, 2, 3, 0]           0   \n","Mjob        [at_home, health, other, services, teacher]           0   \n","Fjob        [teacher, other, services, health, at_home]           0   \n","reason                [course, other, home, reputation]           0   \n","guardian                        [mother, father, other]           0   \n","traveltime                                 [2, 1, 3, 4]           0   \n","studytime                                  [2, 3, 1, 4]           0   \n","failures                                   [0, 3, 2, 1]           0   \n","schoolsup                                     [yes, no]           0   \n","famsup                                        [no, yes]           0   \n","paid                                          [no, yes]           0   \n","activities                                    [no, yes]           0   \n","nursery                                       [yes, no]           0   \n","higher                                        [yes, no]           0   \n","internet                                      [no, yes]           0   \n","romantic                                      [no, yes]           0   \n","famrel                                  [4, 5, 3, 1, 2]           0   \n","freetime                                [3, 2, 4, 1, 5]           0   \n","goout                                   [4, 3, 2, 1, 5]           0   \n","Dalc                                    [1, 2, 5, 3, 4]           0   \n","Walc                                    [1, 3, 2, 4, 5]           0   \n","health                                  [3, 5, 1, 2, 4]           0   \n","absences             [6, 4, 10, 2, 0, 16, 14, 7, 8, 25]           0   \n","G1                 [5, 7, 15, 6, 12, 16, 14, 10, 13, 8]           0   \n","G2                [6, 5, 8, 14, 10, 15, 12, 18, 16, 13]           0   \n","G3                [6, 10, 15, 11, 19, 9, 12, 14, 16, 5]           0   \n","\n","           perc_missing  \n","school              0.0  \n","sex                 0.0  \n","age                 0.0  \n","address             0.0  \n","famsize             0.0  \n","Pstatus             0.0  \n","Medu                0.0  \n","Fedu                0.0  \n","Mjob                0.0  \n","Fjob                0.0  \n","reason              0.0  \n","guardian            0.0  \n","traveltime          0.0  \n","studytime           0.0  \n","failures            0.0  \n","schoolsup           0.0  \n","famsup              0.0  \n","paid                0.0  \n","activities          0.0  \n","nursery             0.0  \n","higher              0.0  \n","internet            0.0  \n","romantic            0.0  \n","famrel              0.0  \n","freetime            0.0  \n","goout               0.0  \n","Dalc                0.0  \n","Walc                0.0  \n","health              0.0  \n","absences            0.0  \n","G1                  0.0  \n","G2                  0.0  \n","G3                  0.0  "]},"metadata":{},"output_type":"display_data"}],"source":["!pip install fast_ml\n","from fast_ml import eda\n","from fast_ml.utilities import display_all\n","\n","df = pd.read_csv('../input/student-performance-data/student_data.csv')\n","\n","# One of the most useful dataframe summary view\n","summary_df = eda.df_info(df)\n","display_all(summary_df)"]},{"cell_type":"markdown","id":"ba34b6bb","metadata":{"papermill":{"duration":0.005095,"end_time":"2022-06-27T19:52:46.06528","exception":false,"start_time":"2022-06-27T19:52:46.060185","status":"completed"},"tags":[]},"source":["the result is a dataframe containing a summary of the dataset in the input including: variables, datatype, number of unique values, sample of unique values, missing count, missing percent"]},{"cell_type":"code","execution_count":3,"id":"a64a8c87","metadata":{"execution":{"iopub.execute_input":"2022-06-27T19:52:46.078227Z","iopub.status.busy":"2022-06-27T19:52:46.077801Z","iopub.status.idle":"2022-06-27T19:52:46.106999Z","shell.execute_reply":"2022-06-27T19:52:46.10561Z"},"papermill":{"duration":0.040017,"end_time":"2022-06-27T19:52:46.110673","exception":false,"start_time":"2022-06-27T19:52:46.070656","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>school</th>\n","      <th>sex</th>\n","      <th>age</th>\n","      <th>address</th>\n","      <th>famsize</th>\n","      <th>Pstatus</th>\n","      <th>Medu</th>\n","      <th>Fedu</th>\n","      <th>Mjob</th>\n","      <th>Fjob</th>\n","      <th>reason</th>\n","      <th>guardian</th>\n","      <th>traveltime</th>\n","      <th>studytime</th>\n","      <th>failures</th>\n","      <th>schoolsup</th>\n","      <th>famsup</th>\n","      <th>paid</th>\n","      <th>activities</th>\n","      <th>nursery</th>\n","      <th>higher</th>\n","      <th>internet</th>\n","      <th>romantic</th>\n","      <th>famrel</th>\n","      <th>freetime</th>\n","      <th>goout</th>\n","      <th>Dalc</th>\n","      <th>Walc</th>\n","      <th>health</th>\n","      <th>absences</th>\n","      <th>G1</th>\n","      <th>G2</th>\n","      <th>G3</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>116</th>\n","      <td>GP</td>\n","      <td>M</td>\n","      <td>15</td>\n","      <td>U</td>\n","      <td>GT3</td>\n","      <td>T</td>\n","      <td>4</td>\n","      <td>4</td>\n","      <td>other</td>\n","      <td>teacher</td>\n","      <td>reputation</td>\n","      <td>father</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>no</td>\n","      <td>yes</td>\n","      <td>no</td>\n","      <td>yes</td>\n","      <td>yes</td>\n","      <td>yes</td>\n","      <td>no</td>\n","      <td>no</td>\n","      <td>4</td>\n","      <td>4</td>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>11</td>\n","      <td>13</td>\n","      <td>14</td>\n","    </tr>\n","    <tr>\n","      <th>245</th>\n","      <td>GP</td>\n","      <td>M</td>\n","      <td>16</td>\n","      <td>U</td>\n","      <td>GT3</td>\n","      <td>T</td>\n","      <td>2</td>\n","      <td>1</td>\n","      <td>other</td>\n","      <td>other</td>\n","      <td>course</td>\n","      <td>mother</td>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>no</td>\n","      <td>no</td>\n","      <td>no</td>\n","      <td>no</td>\n","      <td>yes</td>\n","      <td>yes</td>\n","      <td>yes</td>\n","      <td>no</td>\n","      <td>4</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>6</td>\n","      <td>18</td>\n","      <td>18</td>\n","      <td>18</td>\n","    </tr>\n","    <tr>\n","      <th>125</th>\n","      <td>GP</td>\n","      <td>M</td>\n","      <td>15</td>\n","      <td>U</td>\n","      <td>GT3</td>\n","      <td>T</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>services</td>\n","      <td>services</td>\n","      <td>home</td>\n","      <td>father</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>yes</td>\n","      <td>no</td>\n","      <td>no</td>\n","      <td>no</td>\n","      <td>yes</td>\n","      <td>yes</td>\n","      <td>yes</td>\n","      <td>no</td>\n","      <td>5</td>\n","      <td>5</td>\n","      <td>5</td>\n","      <td>3</td>\n","      <td>2</td>\n","      <td>5</td>\n","      <td>0</td>\n","      <td>13</td>\n","      <td>13</td>\n","      <td>12</td>\n","    </tr>\n","    <tr>\n","      <th>89</th>\n","      <td>GP</td>\n","      <td>M</td>\n","      <td>16</td>\n","      <td>U</td>\n","      <td>LE3</td>\n","      <td>A</td>\n","      <td>4</td>\n","      <td>4</td>\n","      <td>teacher</td>\n","      <td>health</td>\n","      <td>reputation</td>\n","      <td>mother</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>no</td>\n","      <td>yes</td>\n","      <td>no</td>\n","      <td>no</td>\n","      <td>yes</td>\n","      <td>yes</td>\n","      <td>no</td>\n","      <td>no</td>\n","      <td>4</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>5</td>\n","      <td>5</td>\n","      <td>18</td>\n","      <td>8</td>\n","      <td>6</td>\n","      <td>7</td>\n","    </tr>\n","    <tr>\n","      <th>84</th>\n","      <td>GP</td>\n","      <td>F</td>\n","      <td>15</td>\n","      <td>U</td>\n","      <td>GT3</td>\n","      <td>T</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>other</td>\n","      <td>other</td>\n","      <td>home</td>\n","      <td>father</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>no</td>\n","      <td>yes</td>\n","      <td>no</td>\n","      <td>yes</td>\n","      <td>no</td>\n","      <td>yes</td>\n","      <td>yes</td>\n","      <td>no</td>\n","      <td>4</td>\n","      <td>3</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>2</td>\n","      <td>9</td>\n","      <td>10</td>\n","      <td>10</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["    school sex  age address famsize Pstatus  Medu  Fedu      Mjob      Fjob  \\\n","116     GP   M   15       U     GT3       T     4     4     other   teacher   \n","245     GP   M   16       U     GT3       T     2     1     other     other   \n","125     GP   M   15       U     GT3       T     3     4  services  services   \n","89      GP   M   16       U     LE3       A     4     4   teacher    health   \n","84      GP   F   15       U     GT3       T     1     1     other     other   \n","\n","         reason guardian  traveltime  studytime  failures schoolsup famsup  \\\n","116  reputation   father           2          2         0        no    yes   \n","245      course   mother           3          1         0        no     no   \n","125        home   father           1          1         0       yes     no   \n","89   reputation   mother           1          2         0        no    yes   \n","84         home   father           1          2         0        no    yes   \n","\n","    paid activities nursery higher internet romantic  famrel  freetime  goout  \\\n","116   no        yes     yes    yes       no       no       4         4      3   \n","245   no         no     yes    yes      yes       no       4         3      3   \n","125   no         no     yes    yes      yes       no       5         5      5   \n","89    no         no     yes    yes       no       no       4         1      3   \n","84    no        yes      no    yes      yes       no       4         3      2   \n","\n","     Dalc  Walc  health  absences  G1  G2  G3  \n","116     1     1       2         2  11  13  14  \n","245     1     1       4         6  18  18  18  \n","125     3     2       5         0  13  13  12  \n","89      3     5       5        18   8   6   7  \n","84      2     3       4         2   9  10  10  "]},"execution_count":3,"metadata":{},"output_type":"execute_result"}],"source":["pd.set_option('display.max_columns', None)\n","df.sample(5) #train.head(5)"]},{"cell_type":"code","execution_count":4,"id":"3d96b471","metadata":{"execution":{"iopub.execute_input":"2022-06-27T19:52:46.12472Z","iopub.status.busy":"2022-06-27T19:52:46.124044Z","iopub.status.idle":"2022-06-27T19:52:46.207718Z","shell.execute_reply":"2022-06-27T19:52:46.206565Z"},"papermill":{"duration":0.093727,"end_time":"2022-06-27T19:52:46.210292","exception":false,"start_time":"2022-06-27T19:52:46.116565","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>count</th>\n","      <th>mean</th>\n","      <th>std</th>\n","      <th>0%</th>\n","      <th>10%</th>\n","      <th>20%</th>\n","      <th>30%</th>\n","      <th>40%</th>\n","      <th>50%</th>\n","      <th>60%</th>\n","      <th>70%</th>\n","      <th>80%</th>\n","      <th>90%</th>\n","      <th>95%</th>\n","      <th>100%</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>age</th>\n","      <td>395.0</td>\n","      <td>16.696203</td>\n","      <td>1.276043</td>\n","      <td>15.0</td>\n","      <td>15.0</td>\n","      <td>15.0</td>\n","      <td>16.0</td>\n","      <td>16.0</td>\n","      <td>17.0</td>\n","      <td>17.0</td>\n","      <td>17.0</td>\n","      <td>18.0</td>\n","      <td>18.0</td>\n","      <td>19.0</td>\n","      <td>22.0</td>\n","    </tr>\n","    <tr>\n","      <th>Medu</th>\n","      <td>395.0</td>\n","      <td>2.749367</td>\n","      <td>1.094735</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","    </tr>\n","    <tr>\n","      <th>Fedu</th>\n","      <td>395.0</td>\n","      <td>2.521519</td>\n","      <td>1.088201</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","    </tr>\n","    <tr>\n","      <th>traveltime</th>\n","      <td>395.0</td>\n","      <td>1.448101</td>\n","      <td>0.697505</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>3.0</td>\n","      <td>4.0</td>\n","    </tr>\n","    <tr>\n","      <th>studytime</th>\n","      <td>395.0</td>\n","      <td>2.035443</td>\n","      <td>0.839240</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","    </tr>\n","    <tr>\n","      <th>failures</th>\n","      <td>395.0</td>\n","      <td>0.334177</td>\n","      <td>0.743651</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>2.0</td>\n","      <td>3.0</td>\n","    </tr>\n","    <tr>\n","      <th>famrel</th>\n","      <td>395.0</td>\n","      <td>3.944304</td>\n","      <td>0.896659</td>\n","      <td>1.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>5.0</td>\n","      <td>5.0</td>\n","      <td>5.0</td>\n","      <td>5.0</td>\n","    </tr>\n","    <tr>\n","      <th>freetime</th>\n","      <td>395.0</td>\n","      <td>3.235443</td>\n","      <td>0.998862</td>\n","      <td>1.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>4.6</td>\n","      <td>5.0</td>\n","      <td>5.0</td>\n","    </tr>\n","    <tr>\n","      <th>goout</th>\n","      <td>395.0</td>\n","      <td>3.108861</td>\n","      <td>1.113278</td>\n","      <td>1.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>5.0</td>\n","      <td>5.0</td>\n","      <td>5.0</td>\n","    </tr>\n","    <tr>\n","      <th>Dalc</th>\n","      <td>395.0</td>\n","      <td>1.481013</td>\n","      <td>0.890741</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.8</td>\n","      <td>2.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>5.0</td>\n","    </tr>\n","    <tr>\n","      <th>Walc</th>\n","      <td>395.0</td>\n","      <td>2.291139</td>\n","      <td>1.287897</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>3.2</td>\n","      <td>4.0</td>\n","      <td>5.0</td>\n","      <td>5.0</td>\n","    </tr>\n","    <tr>\n","      <th>health</th>\n","      <td>395.0</td>\n","      <td>3.554430</td>\n","      <td>1.390303</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","      <td>2.0</td>\n","      <td>3.0</td>\n","      <td>3.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>5.0</td>\n","      <td>5.0</td>\n","      <td>5.0</td>\n","      <td>5.0</td>\n","      <td>5.0</td>\n","    </tr>\n","    <tr>\n","      <th>absences</th>\n","      <td>395.0</td>\n","      <td>5.708861</td>\n","      <td>8.003096</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>2.0</td>\n","      <td>2.0</td>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>6.0</td>\n","      <td>10.0</td>\n","      <td>14.0</td>\n","      <td>18.3</td>\n","      <td>75.0</td>\n","    </tr>\n","    <tr>\n","      <th>G1</th>\n","      <td>395.0</td>\n","      <td>10.908861</td>\n","      <td>3.319195</td>\n","      <td>3.0</td>\n","      <td>7.0</td>\n","      <td>8.0</td>\n","      <td>9.0</td>\n","      <td>10.0</td>\n","      <td>11.0</td>\n","      <td>12.0</td>\n","      <td>13.0</td>\n","      <td>14.0</td>\n","      <td>16.0</td>\n","      <td>16.0</td>\n","      <td>19.0</td>\n","    </tr>\n","    <tr>\n","      <th>G2</th>\n","      <td>395.0</td>\n","      <td>10.713924</td>\n","      <td>3.761505</td>\n","      <td>0.0</td>\n","      <td>6.0</td>\n","      <td>8.0</td>\n","      <td>9.0</td>\n","      <td>10.0</td>\n","      <td>11.0</td>\n","      <td>12.0</td>\n","      <td>13.0</td>\n","      <td>14.0</td>\n","      <td>15.0</td>\n","      <td>16.3</td>\n","      <td>19.0</td>\n","    </tr>\n","    <tr>\n","      <th>G3</th>\n","      <td>395.0</td>\n","      <td>10.415190</td>\n","      <td>4.581443</td>\n","      <td>0.0</td>\n","      <td>5.0</td>\n","      <td>8.0</td>\n","      <td>9.0</td>\n","      <td>10.0</td>\n","      <td>11.0</td>\n","      <td>12.0</td>\n","      <td>13.0</td>\n","      <td>14.0</td>\n","      <td>15.6</td>\n","      <td>17.0</td>\n","      <td>20.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["            count       mean       std    0%   10%   20%   30%   40%   50%  \\\n","age         395.0  16.696203  1.276043  15.0  15.0  15.0  16.0  16.0  17.0   \n","Medu        395.0   2.749367  1.094735   0.0   1.0   2.0   2.0   2.0   3.0   \n","Fedu        395.0   2.521519  1.088201   0.0   1.0   1.0   2.0   2.0   2.0   \n","traveltime  395.0   1.448101  0.697505   1.0   1.0   1.0   1.0   1.0   1.0   \n","studytime   395.0   2.035443  0.839240   1.0   1.0   1.0   2.0   2.0   2.0   \n","failures    395.0   0.334177  0.743651   0.0   0.0   0.0   0.0   0.0   0.0   \n","famrel      395.0   3.944304  0.896659   1.0   3.0   3.0   4.0   4.0   4.0   \n","freetime    395.0   3.235443  0.998862   1.0   2.0   2.0   3.0   3.0   3.0   \n","goout       395.0   3.108861  1.113278   1.0   2.0   2.0   2.0   3.0   3.0   \n","Dalc        395.0   1.481013  0.890741   1.0   1.0   1.0   1.0   1.0   1.0   \n","Walc        395.0   2.291139  1.287897   1.0   1.0   1.0   1.0   2.0   2.0   \n","health      395.0   3.554430  1.390303   1.0   1.0   2.0   3.0   3.0   4.0   \n","absences    395.0   5.708861  8.003096   0.0   0.0   0.0   2.0   2.0   4.0   \n","G1          395.0  10.908861  3.319195   3.0   7.0   8.0   9.0  10.0  11.0   \n","G2          395.0  10.713924  3.761505   0.0   6.0   8.0   9.0  10.0  11.0   \n","G3          395.0  10.415190  4.581443   0.0   5.0   8.0   9.0  10.0  11.0   \n","\n","             60%   70%   80%   90%   95%  100%  \n","age         17.0  17.0  18.0  18.0  19.0  22.0  \n","Medu         3.0   4.0   4.0   4.0   4.0   4.0  \n","Fedu         3.0   3.0   4.0   4.0   4.0   4.0  \n","traveltime   1.0   2.0   2.0   2.0   3.0   4.0  \n","studytime    2.0   2.0   3.0   3.0   4.0   4.0  \n","failures     0.0   0.0   1.0   1.0   2.0   3.0  \n","famrel       4.0   4.0   5.0   5.0   5.0   5.0  \n","freetime     3.0   4.0   4.0   4.6   5.0   5.0  \n","goout        3.0   4.0   4.0   5.0   5.0   5.0  \n","Dalc         1.0   1.8   2.0   3.0   3.0   5.0  \n","Walc         3.0   3.0   3.2   4.0   5.0   5.0  \n","health       4.0   5.0   5.0   5.0   5.0   5.0  \n","absences     4.0   6.0  10.0  14.0  18.3  75.0  \n","G1          12.0  13.0  14.0  16.0  16.0  19.0  \n","G2          12.0  13.0  14.0  15.0  16.3  19.0  \n","G3          12.0  13.0  14.0  15.6  17.0  20.0  "]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["eda.numerical_describe(df) #descriptive statistics for numerical variables"]},{"cell_type":"code","execution_count":5,"id":"80895a1a","metadata":{"execution":{"iopub.execute_input":"2022-06-27T19:52:46.225705Z","iopub.status.busy":"2022-06-27T19:52:46.224899Z","iopub.status.idle":"2022-06-27T19:52:46.237947Z","shell.execute_reply":"2022-06-27T19:52:46.236362Z"},"papermill":{"duration":0.023431,"end_time":"2022-06-27T19:52:46.240266","exception":true,"start_time":"2022-06-27T19:52:46.216835","status":"failed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Total Number of observations :  395\n","\n","Datatype : int64\n","\n","Number of distinct values : 4\n","\n"," Number of unique values in this variable is less then threshold value of 20\n","\n"," Consider using Categorical EDA\n"]},{"ename":"SystemExit","evalue":"","output_type":"error","traceback":["An exception has occurred, use %tb to see the full traceback.\n","\u001b[0;31mSystemExit\u001b[0m\n"]},{"name":"stderr","output_type":"stream","text":["/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3560: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n","  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"]}],"source":["eda.numerical_variable_detail(df, variable='studytime') #description of a single variable "]},{"cell_type":"code","execution_count":null,"id":"5e2d967c","metadata":{"execution":{"iopub.execute_input":"2022-06-27T19:51:51.561552Z","iopub.status.busy":"2022-06-27T19:51:51.560426Z","iopub.status.idle":"2022-06-27T19:51:52.363458Z","shell.execute_reply":"2022-06-27T19:51:52.360769Z","shell.execute_reply.started":"2022-06-27T19:51:51.561497Z"},"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"source":["eda.numerical_plots(df, variables=['freetime', 'studytime','failures' ]) #uni-variate plots"]},{"cell_type":"code","execution_count":null,"id":"19308ed8","metadata":{"execution":{"iopub.execute_input":"2022-06-27T19:51:55.223526Z","iopub.status.busy":"2022-06-27T19:51:55.223137Z","iopub.status.idle":"2022-06-27T19:51:55.257174Z","shell.execute_reply":"2022-06-27T19:51:55.255997Z","shell.execute_reply.started":"2022-06-27T19:51:55.223497Z"},"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"source":["mean_grade= (df.G1 + df.G2 + df.G3)/3\n","df['mean_G']= mean_grade\n","df.head(5)"]},{"cell_type":"code","execution_count":null,"id":"ab46be2d","metadata":{"execution":{"iopub.execute_input":"2022-06-27T19:51:57.280631Z","iopub.status.busy":"2022-06-27T19:51:57.280243Z","iopub.status.idle":"2022-06-27T19:51:58.304325Z","shell.execute_reply":"2022-06-27T19:51:58.303106Z","shell.execute_reply.started":"2022-06-27T19:51:57.280602Z"},"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"source":["eda.numerical_plots_with_target(df, variables=['freetime', 'studytime','failures' ], target='mean_G',model_type='reg')\n","#Scatter plot of all the numerical variables provided as input with target."]},{"cell_type":"code","execution_count":null,"id":"63138b51","metadata":{"execution":{"iopub.execute_input":"2022-06-27T19:52:00.066186Z","iopub.status.busy":"2022-06-27T19:52:00.065819Z","iopub.status.idle":"2022-06-27T19:52:00.103471Z","shell.execute_reply":"2022-06-27T19:52:00.102306Z","shell.execute_reply.started":"2022-06-27T19:52:00.066157Z"},"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"source":["eda.numerical_check_outliers(df)"]},{"cell_type":"code","execution_count":null,"id":"92dedada","metadata":{"execution":{"iopub.execute_input":"2022-06-27T19:52:02.832034Z","iopub.status.busy":"2022-06-27T19:52:02.831657Z","iopub.status.idle":"2022-06-27T19:52:03.203494Z","shell.execute_reply":"2022-06-27T19:52:03.202273Z","shell.execute_reply.started":"2022-06-27T19:52:02.832009Z"},"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"source":["from fast_ml.model_development import train_valid_test_split\n","\n","#X_train, y_train, X_valid, y_valid, X_test, y_test = train_valid_test_split(df, target = df['mean_G'], train_size=0.8, valid_size=0.1, test_size=0.1)\n","\n","# Get the shape of all the datasets\n","#print(X_train.shape), print(y_train.shape)\n","#print(X_valid.shape), print(y_valid.shape)\n","#print(X_test.shape), print(y_test.shape)"]},{"cell_type":"code","execution_count":null,"id":"29bc510c","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"pending"},"tags":[]},"outputs":[],"source":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"},"papermill":{"default_parameters":{},"duration":26.305882,"end_time":"2022-06-27T19:52:47.271003","environment_variables":{},"exception":true,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2022-06-27T19:52:20.965121","version":"2.3.4"}},"nbformat":4,"nbformat_minor":5}