data-cleaning/data_cleaning_using_klib.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e4e496b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Installing library\n",
    "\n",
    "!pip install klib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d425d6a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Importing libraries\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import klib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "23a68697",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Loading Data\n",
    "\n",
    "# Data URL from UCI repository\n",
    "auto_data_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/autos/imports-85.data'\n",
    "\n",
    "# List of columns\n",
    "column_names = ['symboling', 'normalized-losses', 'make', 'fuel-type', 'aspiration', 'num-of-doors', 'body-style', \n",
    "                'drive-wheels', 'engine-location', 'wheel-base', 'length', 'width', 'height', 'curb-weight', \n",
    "                'engine-type', 'num-of-cylinders', 'engine-size', 'fuel-system', 'bore', 'stroke', 'compression-ratio', \n",
    "                'horsepower', 'peak-rpm', 'city-mpg', 'highway-mpg', 'price']\n",
    "\n",
    "# Reading data as pandas dataframe\n",
    "df = pd.read_csv(auto_data_url, names=column_names)\n",
    "df = df.replace({'?': np.nan})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "77f0e3b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Converting columns to Numeric type\n",
    "numeric_cols = ['normalized-losses', 'engine-size', 'horsepower', 'peak-rpm', 'bore', 'stroke', 'price']\n",
    "df[numeric_cols] = df[numeric_cols].apply(pd.to_numeric, errors='coerce', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a3ca32fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 205 entries, 0 to 204\n",
      "Data columns (total 26 columns):\n",
      " #   Column             Non-Null Count  Dtype  \n",
      "---  ------             --------------  -----  \n",
      " 0   symboling          205 non-null    int64  \n",
      " 1   normalized-losses  164 non-null    float64\n",
      " 2   make               205 non-null    object \n",
      " 3   fuel-type          205 non-null    object \n",
      " 4   aspiration         205 non-null    object \n",
      " 5   num-of-doors       203 non-null    object \n",
      " 6   body-style         205 non-null    object \n",
      " 7   drive-wheels       205 non-null    object \n",
      " 8   engine-location    205 non-null    object \n",
      " 9   wheel-base         205 non-null    float64\n",
      " 10  length             205 non-null    float64\n",
      " 11  width              205 non-null    float64\n",
      " 12  height             205 non-null    float64\n",
      " 13  curb-weight        205 non-null    int64  \n",
      " 14  engine-type        205 non-null    object \n",
      " 15  num-of-cylinders   205 non-null    object \n",
      " 16  engine-size        205 non-null    float64\n",
      " 17  fuel-system        205 non-null    object \n",
      " 18  bore               201 non-null    float64\n",
      " 19  stroke             201 non-null    float64\n",
      " 20  compression-ratio  205 non-null    float64\n",
      " 21  horsepower         203 non-null    float64\n",
      " 22  peak-rpm           203 non-null    float64\n",
      " 23  city-mpg           205 non-null    int64  \n",
      " 24  highway-mpg        205 non-null    int64  \n",
      " 25  price              201 non-null    float64\n",
      "dtypes: float64(12), int64(4), object(10)\n",
      "memory usage: 41.8+ KB\n"
     ]
    }
   ],
   "source": [
    "## Getting datafrmae columns information\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6d789a21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['symboling', 'normalized-losses', 'make', 'fuel-type', 'aspiration',\n",
       "       'num-of-doors', 'body-style', 'drive-wheels', 'engine-location',\n",
       "       'wheel-base', 'length', 'width', 'height', 'curb-weight', 'engine-type',\n",
       "       'num-of-cylinders', 'engine-size', 'fuel-system', 'bore', 'stroke',\n",
       "       'compression-ratio', 'horsepower', 'peak-rpm', 'city-mpg',\n",
       "       'highway-mpg', 'price'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Getting list of columns in a dataframe\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e05f205d",
   "metadata": {},
   "source": [
    "## klib.clean - functions for cleaning datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4abcc7d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of cleaned data: (205, 26)Remaining NAs: 59\n",
      "\n",
      "Changes:\n",
      "Dropped rows: 0\n",
      "     of which 0 duplicates. (Rows: [])\n",
      "Dropped columns: 0\n",
      "     of which 0 single valued.     Columns: []\n",
      "Dropped missing values: 0\n",
      "Reduced memory by at least: 0.02 MB (-50.0%)\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 205 entries, 0 to 204\n",
      "Data columns (total 26 columns):\n",
      " #   Column             Non-Null Count  Dtype   \n",
      "---  ------             --------------  -----   \n",
      " 0   symboling          205 non-null    int8    \n",
      " 1   normalized_losses  164 non-null    Float32 \n",
      " 2   make               205 non-null    string  \n",
      " 3   fuel_type          205 non-null    category\n",
      " 4   aspiration         205 non-null    category\n",
      " 5   num_of_doors       203 non-null    category\n",
      " 6   body_style         205 non-null    category\n",
      " 7   drive_wheels       205 non-null    category\n",
      " 8   engine_location    205 non-null    category\n",
      " 9   wheel_base         205 non-null    Float32 \n",
      " 10  length             205 non-null    Float32 \n",
      " 11  width              205 non-null    Float32 \n",
      " 12  height             205 non-null    Float32 \n",
      " 13  curb_weight        205 non-null    int16   \n",
      " 14  engine_type        205 non-null    string  \n",
      " 15  num_of_cylinders   205 non-null    string  \n",
      " 16  engine_size        205 non-null    Float32 \n",
      " 17  fuel_system        205 non-null    string  \n",
      " 18  bore               201 non-null    Float32 \n",
      " 19  stroke             201 non-null    Float32 \n",
      " 20  compression_ratio  205 non-null    Float32 \n",
      " 21  horsepower         203 non-null    Float32 \n",
      " 22  peak_rpm           203 non-null    Float32 \n",
      " 23  city_mpg           205 non-null    int8    \n",
      " 24  highway_mpg        205 non-null    int8    \n",
      " 25  price              201 non-null    Float32 \n",
      "dtypes: Float32(12), category(6), int16(1), int8(3), string(4)\n",
      "memory usage: 21.6 KB\n"
     ]
    }
   ],
   "source": [
    "## Cleaning dataframe\n",
    "## 1. Remove Duplicates\n",
    "## 2. Remove Empty Rows and Columns\n",
    "## 3. Remove columns having a single/constant value\n",
    "## 3. Adjust column data types (to reduce memory of a dataframe)\n",
    "\n",
    "df_cleaned = klib.data_cleaning(df) # performs datacleaning (drop duplicates & empty rows/cols, adjust dtypes,...)\n",
    "df_cleaned.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1b541bcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['symboling', 'normalized_losses', 'make', 'fuel_type', 'aspiration',\n",
       "       'num_of_doors', 'body_style', 'drive_wheels', 'engine_location',\n",
       "       'wheel_base', 'length', 'width', 'height', 'curb_weight', 'engine_type',\n",
       "       'num_of_cylinders', 'engine_size', 'fuel_system', 'bore', 'stroke',\n",
       "       'compression_ratio', 'horsepower', 'peak_rpm', 'city_mpg',\n",
       "       'highway_mpg', 'price'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## If we want, we can separately call the function for standardizing the column names\n",
    "## By default it can be taken care in 'data_cleaning()' function\n",
    "\n",
    "df_changed_column_names = klib.clean_column_names(df) # cleans and standardizes column names, also called inside data_cleaning()\n",
    "df_changed_column_names.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "446df42f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 205 entries, 0 to 204\n",
      "Data columns (total 26 columns):\n",
      " #   Column             Non-Null Count  Dtype   \n",
      "---  ------             --------------  -----   \n",
      " 0   symboling          205 non-null    int8    \n",
      " 1   normalized_losses  164 non-null    Float32 \n",
      " 2   make               205 non-null    string  \n",
      " 3   fuel_type          205 non-null    category\n",
      " 4   aspiration         205 non-null    category\n",
      " 5   num_of_doors       203 non-null    category\n",
      " 6   body_style         205 non-null    category\n",
      " 7   drive_wheels       205 non-null    category\n",
      " 8   engine_location    205 non-null    category\n",
      " 9   wheel_base         205 non-null    Float32 \n",
      " 10  length             205 non-null    Float32 \n",
      " 11  width              205 non-null    Float32 \n",
      " 12  height             205 non-null    Float32 \n",
      " 13  curb_weight        205 non-null    int16   \n",
      " 14  engine_type        205 non-null    category\n",
      " 15  num_of_cylinders   205 non-null    category\n",
      " 16  engine_size        205 non-null    Float32 \n",
      " 17  fuel_system        205 non-null    category\n",
      " 18  bore               201 non-null    Float32 \n",
      " 19  stroke             201 non-null    Float32 \n",
      " 20  compression_ratio  205 non-null    Float32 \n",
      " 21  horsepower         203 non-null    Float32 \n",
      " 22  peak_rpm           203 non-null    Float32 \n",
      " 23  city_mpg           205 non-null    int8    \n",
      " 24  highway_mpg        205 non-null    int8    \n",
      " 25  price              201 non-null    Float32 \n",
      "dtypes: Float32(12), category(9), int16(1), int8(3), string(1)\n",
      "memory usage: 18.4 KB\n"
     ]
    }
   ],
   "source": [
    "## If we want, we can separately call the function for converting the column data type\n",
    "## By default it can be taken care in 'data_cleaning()' function\n",
    "\n",
    "df_changed_data_types = klib.convert_datatypes(df) # converts existing to more efficient dtypes, also called inside data_cleaning()\n",
    "df_changed_data_types.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3f960efd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(205, 26)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## If we want, we can separately call the function for dropping the missing values\n",
    "## By default it can be taken care in 'data_cleaning()' function\n",
    "\n",
    "df_subset_missing = klib.drop_missing(df) # drops missing values, also called in data_cleaning()\n",
    "df_subset_missing.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a4ad46da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(205, 26)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Dropping columns having high ratio of missing values\n",
    "\n",
    "df_subset_columns = klib.mv_col_handling(df) # drops features with high ratio of missing vals based on informational content\n",
    "df_subset_columns.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a22e4652",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>body_style</th>\n",
       "      <th>curb_weight</th>\n",
       "      <th>city_mpg</th>\n",
       "      <th>highway_mpg</th>\n",
       "      <th>price</th>\n",
       "      <th>pooled_vars</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>convertible</td>\n",
       "      <td>2548</td>\n",
       "      <td>21</td>\n",
       "      <td>27</td>\n",
       "      <td>13495.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>convertible</td>\n",
       "      <td>2548</td>\n",
       "      <td>21</td>\n",
       "      <td>27</td>\n",
       "      <td>16500.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>2823</td>\n",
       "      <td>19</td>\n",
       "      <td>26</td>\n",
       "      <td>16500.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2337</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>13950.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2824</td>\n",
       "      <td>18</td>\n",
       "      <td>22</td>\n",
       "      <td>17450.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2507</td>\n",
       "      <td>19</td>\n",
       "      <td>25</td>\n",
       "      <td>15250.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2844</td>\n",
       "      <td>19</td>\n",
       "      <td>25</td>\n",
       "      <td>17710.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>wagon</td>\n",
       "      <td>2954</td>\n",
       "      <td>19</td>\n",
       "      <td>25</td>\n",
       "      <td>18920.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>sedan</td>\n",
       "      <td>3086</td>\n",
       "      <td>17</td>\n",
       "      <td>20</td>\n",
       "      <td>23875.0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>3053</td>\n",
       "      <td>16</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2395</td>\n",
       "      <td>23</td>\n",
       "      <td>29</td>\n",
       "      <td>16430.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2395</td>\n",
       "      <td>23</td>\n",
       "      <td>29</td>\n",
       "      <td>16925.0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2710</td>\n",
       "      <td>21</td>\n",
       "      <td>28</td>\n",
       "      <td>20970.0</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2765</td>\n",
       "      <td>21</td>\n",
       "      <td>28</td>\n",
       "      <td>21105.0</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>sedan</td>\n",
       "      <td>3055</td>\n",
       "      <td>20</td>\n",
       "      <td>25</td>\n",
       "      <td>24565.0</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>sedan</td>\n",
       "      <td>3230</td>\n",
       "      <td>16</td>\n",
       "      <td>22</td>\n",
       "      <td>30760.0</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>sedan</td>\n",
       "      <td>3380</td>\n",
       "      <td>16</td>\n",
       "      <td>22</td>\n",
       "      <td>41315.0</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>sedan</td>\n",
       "      <td>3505</td>\n",
       "      <td>15</td>\n",
       "      <td>20</td>\n",
       "      <td>36880.0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>1488</td>\n",
       "      <td>47</td>\n",
       "      <td>53</td>\n",
       "      <td>5151.0</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>1874</td>\n",
       "      <td>38</td>\n",
       "      <td>43</td>\n",
       "      <td>6295.0</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>sedan</td>\n",
       "      <td>1909</td>\n",
       "      <td>38</td>\n",
       "      <td>43</td>\n",
       "      <td>6575.0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>1876</td>\n",
       "      <td>37</td>\n",
       "      <td>41</td>\n",
       "      <td>5572.0</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>1876</td>\n",
       "      <td>31</td>\n",
       "      <td>38</td>\n",
       "      <td>6377.0</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>2128</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>7957.0</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>1967</td>\n",
       "      <td>31</td>\n",
       "      <td>38</td>\n",
       "      <td>6229.0</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>sedan</td>\n",
       "      <td>1989</td>\n",
       "      <td>31</td>\n",
       "      <td>38</td>\n",
       "      <td>6692.0</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>sedan</td>\n",
       "      <td>1989</td>\n",
       "      <td>31</td>\n",
       "      <td>38</td>\n",
       "      <td>7609.0</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2191</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>8558.0</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>wagon</td>\n",
       "      <td>2535</td>\n",
       "      <td>24</td>\n",
       "      <td>30</td>\n",
       "      <td>8921.0</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>2811</td>\n",
       "      <td>19</td>\n",
       "      <td>24</td>\n",
       "      <td>12964.0</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>1713</td>\n",
       "      <td>49</td>\n",
       "      <td>54</td>\n",
       "      <td>6479.0</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>1819</td>\n",
       "      <td>31</td>\n",
       "      <td>38</td>\n",
       "      <td>6855.0</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>1837</td>\n",
       "      <td>38</td>\n",
       "      <td>42</td>\n",
       "      <td>5399.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>1940</td>\n",
       "      <td>30</td>\n",
       "      <td>34</td>\n",
       "      <td>6529.0</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>1956</td>\n",
       "      <td>30</td>\n",
       "      <td>34</td>\n",
       "      <td>7129.0</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2010</td>\n",
       "      <td>30</td>\n",
       "      <td>34</td>\n",
       "      <td>7295.0</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>wagon</td>\n",
       "      <td>2024</td>\n",
       "      <td>30</td>\n",
       "      <td>34</td>\n",
       "      <td>7295.0</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>2236</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>7895.0</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>2289</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>9095.0</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2304</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>8845.0</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2372</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>10295.0</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2465</td>\n",
       "      <td>24</td>\n",
       "      <td>28</td>\n",
       "      <td>12945.0</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2293</td>\n",
       "      <td>25</td>\n",
       "      <td>31</td>\n",
       "      <td>10345.0</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>sedan</td>\n",
       "      <td>2337</td>\n",
       "      <td>24</td>\n",
       "      <td>29</td>\n",
       "      <td>6785.0</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>sedan</td>\n",
       "      <td>1874</td>\n",
       "      <td>38</td>\n",
       "      <td>43</td>\n",
       "      <td>NaN</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>sedan</td>\n",
       "      <td>1909</td>\n",
       "      <td>38</td>\n",
       "      <td>43</td>\n",
       "      <td>NaN</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>hatchback</td>\n",
       "      <td>2734</td>\n",
       "      <td>24</td>\n",
       "      <td>29</td>\n",
       "      <td>11048.0</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>sedan</td>\n",
       "      <td>4066</td>\n",
       "      <td>15</td>\n",
       "      <td>19</td>\n",
       "      <td>32250.0</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>sedan</td>\n",
       "      <td>4066</td>\n",
       "      <td>15</td>\n",
       "      <td>19</td>\n",
       "      <td>35550.0</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>sedan</td>\n",
       "      <td>3950</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>36000.0</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     body_style  curb_weight  city_mpg  highway_mpg    price  pooled_vars\n",
       "0   convertible         2548        21           27  13495.0            0\n",
       "1   convertible         2548        21           27  16500.0            0\n",
       "2     hatchback         2823        19           26  16500.0            2\n",
       "3         sedan         2337        24           30  13950.0            3\n",
       "4         sedan         2824        18           22  17450.0            4\n",
       "5         sedan         2507        19           25  15250.0            5\n",
       "6         sedan         2844        19           25  17710.0            6\n",
       "7         wagon         2954        19           25  18920.0            7\n",
       "8         sedan         3086        17           20  23875.0            8\n",
       "9     hatchback         3053        16           22      NaN            9\n",
       "10        sedan         2395        23           29  16430.0           10\n",
       "11        sedan         2395        23           29  16925.0           11\n",
       "12        sedan         2710        21           28  20970.0           12\n",
       "13        sedan         2765        21           28  21105.0           13\n",
       "14        sedan         3055        20           25  24565.0           14\n",
       "15        sedan         3230        16           22  30760.0           15\n",
       "16        sedan         3380        16           22  41315.0           16\n",
       "17        sedan         3505        15           20  36880.0           17\n",
       "18    hatchback         1488        47           53   5151.0           18\n",
       "19    hatchback         1874        38           43   6295.0           19\n",
       "20        sedan         1909        38           43   6575.0           20\n",
       "21    hatchback         1876        37           41   5572.0           21\n",
       "22    hatchback         1876        31           38   6377.0           22\n",
       "23    hatchback         2128        24           30   7957.0           23\n",
       "24    hatchback         1967        31           38   6229.0           24\n",
       "25        sedan         1989        31           38   6692.0           24\n",
       "26        sedan         1989        31           38   7609.0           24\n",
       "27        sedan         2191        24           30   8558.0           27\n",
       "28        wagon         2535        24           30   8921.0           28\n",
       "29    hatchback         2811        19           24  12964.0           29\n",
       "30    hatchback         1713        49           54   6479.0           30\n",
       "31    hatchback         1819        31           38   6855.0           31\n",
       "32    hatchback         1837        38           42   5399.0           32\n",
       "33    hatchback         1940        30           34   6529.0           33\n",
       "34    hatchback         1956        30           34   7129.0           33\n",
       "35        sedan         2010        30           34   7295.0           35\n",
       "36        wagon         2024        30           34   7295.0           36\n",
       "37    hatchback         2236        27           33   7895.0           37\n",
       "38    hatchback         2289        27           33   9095.0           37\n",
       "39        sedan         2304        27           33   8845.0           39\n",
       "40        sedan         2372        27           33  10295.0           40\n",
       "41        sedan         2465        24           28  12945.0           41\n",
       "42        sedan         2293        25           31  10345.0           42\n",
       "43        sedan         2337        24           29   6785.0           43\n",
       "44        sedan         1874        38           43      NaN           44\n",
       "45        sedan         1909        38           43      NaN           45\n",
       "46    hatchback         2734        24           29  11048.0           46\n",
       "47        sedan         4066        15           19  32250.0           47\n",
       "48        sedan         4066        15           19  35550.0           48\n",
       "49        sedan         3950        13           17  36000.0           49"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Identify duplicate subset of columns\n",
    "df_column_subset_duplicates = klib.pool_duplicate_subsets(df) # pools subset of cols based on duplicates with min. loss of information\n",
    "df_column_subset_duplicates.head(50)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}