1. EDA - Exploratory Data Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EDA - Exploratory Data Analysis\n",
    "\n",
    "1) Quickly describe a dataset; number of rows/columns, missing data, data types, preview.\n",
    "\n",
    "2) Clean corrupted data; handle missing data, invalid data types, incorrect values.\n",
    "\n",
    "3) Visualize data distributions; bar charts, histograms, box plots.\n",
    "\n",
    "4) Calculate and visualize correlations (relationships) between variables; heat map.\n",
    "\n",
    "Sources:\n",
    "https://www.datacamp.com/community/tutorials/exploratory-data-analysis-python\n",
    "\n",
    "https://towardsdatascience.com/exploratory-data-analysis-in-python-c9a77dfa39ce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>SepalLengthCm</th>\n",
       "      <th>SepalWidthCm</th>\n",
       "      <th>PetalLengthCm</th>\n",
       "      <th>PetalWidthCm</th>\n",
       "      <th>Species</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>146</td>\n",
       "      <td>6.7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.3</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>147</td>\n",
       "      <td>6.3</td>\n",
       "      <td>2.5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.9</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>148</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>149</td>\n",
       "      <td>6.2</td>\n",
       "      <td>3.4</td>\n",
       "      <td>5.4</td>\n",
       "      <td>2.3</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>150</td>\n",
       "      <td>5.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.1</td>\n",
       "      <td>1.8</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Id  SepalLengthCm  SepalWidthCm  PetalLengthCm  PetalWidthCm  \\\n",
       "145  146            6.7           3.0            5.2           2.3   \n",
       "146  147            6.3           2.5            5.0           1.9   \n",
       "147  148            6.5           3.0            5.2           2.0   \n",
       "148  149            6.2           3.4            5.4           2.3   \n",
       "149  150            5.9           3.0            5.1           1.8   \n",
       "\n",
       "            Species  \n",
       "145  Iris-virginica  \n",
       "146  Iris-virginica  \n",
       "147  Iris-virginica  \n",
       "148  Iris-virginica  \n",
       "149  Iris-virginica  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris = pd.read_csv('Data/EDA/iris.csv')\n",
    "#iris.head()\n",
    "iris.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>SepalLengthCm</th>\n",
       "      <th>SepalWidthCm</th>\n",
       "      <th>PetalLengthCm</th>\n",
       "      <th>PetalWidthCm</th>\n",
       "      <th>Species</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>54</td>\n",
       "      <td>5.5</td>\n",
       "      <td>2.3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.3</td>\n",
       "      <td>Iris-versicolor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>82</td>\n",
       "      <td>5.5</td>\n",
       "      <td>2.4</td>\n",
       "      <td>3.7</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Iris-versicolor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>136</td>\n",
       "      <td>7.7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.1</td>\n",
       "      <td>2.3</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>141</td>\n",
       "      <td>6.7</td>\n",
       "      <td>3.1</td>\n",
       "      <td>5.6</td>\n",
       "      <td>2.4</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>86</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1.6</td>\n",
       "      <td>Iris-versicolor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Id  SepalLengthCm  SepalWidthCm  PetalLengthCm  PetalWidthCm  \\\n",
       "53    54            5.5           2.3            4.0           1.3   \n",
       "81    82            5.5           2.4            3.7           1.0   \n",
       "135  136            7.7           3.0            6.1           2.3   \n",
       "140  141            6.7           3.1            5.6           2.4   \n",
       "85    86            6.0           3.4            4.5           1.6   \n",
       "\n",
       "             Species  \n",
       "53   Iris-versicolor  \n",
       "81   Iris-versicolor  \n",
       "135   Iris-virginica  \n",
       "140   Iris-virginica  \n",
       "85   Iris-versicolor  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      Id  SepalLengthCm  SepalWidthCm  PetalLengthCm  PetalWidthCm  \\\n",
      "62    63            6.0           2.2            4.0           1.0   \n",
      "137  138            6.4           3.1            5.5           1.8   \n",
      "69    70            5.6           2.5            3.9           1.1   \n",
      "86    87            6.7           3.1            4.7           1.5   \n",
      "97    98            6.2           2.9            4.3           1.3   \n",
      "\n",
      "             Species  \n",
      "62   Iris-versicolor  \n",
      "137   Iris-virginica  \n",
      "69   Iris-versicolor  \n",
      "86   Iris-versicolor  \n",
      "97   Iris-versicolor  \n"
     ]
    }
   ],
   "source": [
    "from random import sample\n",
    "randomIndex = np.array(sample(range(len(iris)), 5))\n",
    "irisSample = iris.iloc[randomIndex]\n",
    "print(irisSample)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data description\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 6)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Id', 'SepalLengthCm', 'SepalWidthCm', 'PetalLengthCm', 'PetalWidthCm',\n",
      "       'Species'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(iris.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id                 int64\n",
       "SepalLengthCm    float64\n",
       "SepalWidthCm     float64\n",
       "PetalLengthCm    float64\n",
       "PetalWidthCm     float64\n",
       "Species           object\n",
       "dtype: object"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>SepalLengthCm</th>\n",
       "      <th>SepalWidthCm</th>\n",
       "      <th>PetalLengthCm</th>\n",
       "      <th>PetalWidthCm</th>\n",
       "      <th>Species</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>150.000000</td>\n",
       "      <td>150.000000</td>\n",
       "      <td>150.000000</td>\n",
       "      <td>150.000000</td>\n",
       "      <td>150.000000</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>75.500000</td>\n",
       "      <td>5.843333</td>\n",
       "      <td>3.054000</td>\n",
       "      <td>3.758667</td>\n",
       "      <td>1.198667</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>43.445368</td>\n",
       "      <td>0.828066</td>\n",
       "      <td>0.433594</td>\n",
       "      <td>1.764420</td>\n",
       "      <td>0.763161</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.300000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>38.250000</td>\n",
       "      <td>5.100000</td>\n",
       "      <td>2.800000</td>\n",
       "      <td>1.600000</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>75.500000</td>\n",
       "      <td>5.800000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.350000</td>\n",
       "      <td>1.300000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>112.750000</td>\n",
       "      <td>6.400000</td>\n",
       "      <td>3.300000</td>\n",
       "      <td>5.100000</td>\n",
       "      <td>1.800000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>150.000000</td>\n",
       "      <td>7.900000</td>\n",
       "      <td>4.400000</td>\n",
       "      <td>6.900000</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Id  SepalLengthCm  SepalWidthCm  PetalLengthCm  PetalWidthCm  \\\n",
       "count   150.000000     150.000000    150.000000     150.000000    150.000000   \n",
       "unique         NaN            NaN           NaN            NaN           NaN   \n",
       "top            NaN            NaN           NaN            NaN           NaN   \n",
       "freq           NaN            NaN           NaN            NaN           NaN   \n",
       "mean     75.500000       5.843333      3.054000       3.758667      1.198667   \n",
       "std      43.445368       0.828066      0.433594       1.764420      0.763161   \n",
       "min       1.000000       4.300000      2.000000       1.000000      0.100000   \n",
       "25%      38.250000       5.100000      2.800000       1.600000      0.300000   \n",
       "50%      75.500000       5.800000      3.000000       4.350000      1.300000   \n",
       "75%     112.750000       6.400000      3.300000       5.100000      1.800000   \n",
       "max     150.000000       7.900000      4.400000       6.900000      2.500000   \n",
       "\n",
       "               Species  \n",
       "count              150  \n",
       "unique               3  \n",
       "top     Iris-virginica  \n",
       "freq                50  \n",
       "mean               NaN  \n",
       "std                NaN  \n",
       "min                NaN  \n",
       "25%                NaN  \n",
       "50%                NaN  \n",
       "75%                NaN  \n",
       "max                NaN  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris.describe()\n",
    "#see a summary of the training dataset\n",
    "iris.describe(include = \"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150 entries, 0 to 149\n",
      "Data columns (total 6 columns):\n",
      "Id               150 non-null int64\n",
      "SepalLengthCm    150 non-null float64\n",
      "SepalWidthCm     150 non-null float64\n",
      "PetalLengthCm    150 non-null float64\n",
      "PetalWidthCm     150 non-null float64\n",
      "Species          150 non-null object\n",
      "dtypes: float64(4), int64(1), object(1)\n",
      "memory usage: 7.1+ KB\n"
     ]
    }
   ],
   "source": [
    "iris.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Iris-virginica     50\n",
       "Iris-setosa        50\n",
       "Iris-versicolor    50\n",
       "Name: Species, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris['Species'].value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SepalWidthCm\n",
       "2.0     1\n",
       "2.2     3\n",
       "2.3     4\n",
       "2.4     3\n",
       "2.5     8\n",
       "2.6     5\n",
       "2.7     9\n",
       "2.8    14\n",
       "2.9    10\n",
       "3.0    26\n",
       "3.1    12\n",
       "3.2    13\n",
       "3.3     6\n",
       "3.4    12\n",
       "3.5     6\n",
       "3.6     3\n",
       "3.7     3\n",
       "3.8     6\n",
       "3.9     2\n",
       "4.0     1\n",
       "4.1     1\n",
       "4.2     1\n",
       "4.4     1\n",
       "Name: Species, dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris.groupby(['SepalWidthCm'])['Species'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Species          SepalWidthCm\n",
       "Iris-setosa      2.3              1\n",
       "                 2.9              1\n",
       "                 3.0              6\n",
       "                 3.1              5\n",
       "                 3.2              5\n",
       "                 3.3              2\n",
       "                 3.4              9\n",
       "                 3.5              6\n",
       "                 3.6              2\n",
       "                 3.7              3\n",
       "                 3.8              4\n",
       "                 3.9              2\n",
       "                 4.0              1\n",
       "                 4.1              1\n",
       "                 4.2              1\n",
       "                 4.4              1\n",
       "Iris-versicolor  2.0              1\n",
       "                 2.2              2\n",
       "                 2.3              3\n",
       "                 2.4              3\n",
       "                 2.5              4\n",
       "                 2.6              3\n",
       "                 2.7              5\n",
       "                 2.8              6\n",
       "                 2.9              7\n",
       "                 3.0              8\n",
       "                 3.1              3\n",
       "                 3.2              3\n",
       "                 3.3              1\n",
       "                 3.4              1\n",
       "Iris-virginica   2.2              1\n",
       "                 2.5              4\n",
       "                 2.6              2\n",
       "                 2.7              4\n",
       "                 2.8              8\n",
       "                 2.9              2\n",
       "                 3.0             12\n",
       "                 3.1              4\n",
       "                 3.2              5\n",
       "                 3.3              3\n",
       "                 3.4              2\n",
       "                 3.6              1\n",
       "                 3.8              2\n",
       "Name: Species, dtype: int64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris.groupby(['Species','SepalWidthCm'])['Species'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data cleaning\n",
    "\n",
    "### Melbourne housing data\n",
    "\n",
    "Suburb: Suburb\n",
    "\n",
    "Address: Address\n",
    "\n",
    "Rooms: Number of rooms\n",
    "\n",
    "Price: Price in Australian dollars\n",
    "\n",
    "Method:\n",
    "S - property sold;\n",
    "SP - property sold prior;\n",
    "PI - property passed in;\n",
    "PN - sold prior not disclosed;\n",
    "SN - sold not disclosed;\n",
    "NB - no bid;\n",
    "VB - vendor bid;\n",
    "W - withdrawn prior to auction;\n",
    "SA - sold after auction;\n",
    "SS - sold after auction price not disclosed.\n",
    "N/A - price or highest bid not available.\n",
    "\n",
    "Type:\n",
    "br - bedroom(s);\n",
    "h - house,cottage,villa, semi,terrace;\n",
    "u - unit, duplex;\n",
    "t - townhouse;\n",
    "dev site - development site;\n",
    "o res - other residential.\n",
    "\n",
    "SellerG: Real Estate Agent\n",
    "\n",
    "Date: Date sold\n",
    "\n",
    "Distance: Distance from CBD in Kilometres\n",
    "\n",
    "Regionname: General Region (West, North West, North, North east …etc)\n",
    "\n",
    "Propertycount: Number of properties that exist in the suburb.\n",
    "\n",
    "Bedroom2 : Scraped # of Bedrooms (from different source)\n",
    "\n",
    "Bathroom: Number of Bathrooms\n",
    "\n",
    "Car: Number of carspots\n",
    "\n",
    "Landsize: Land Size in Metres\n",
    "\n",
    "BuildingArea: Building Size in Metres\n",
    "\n",
    "YearBuilt: Year the house was built\n",
    "\n",
    "CouncilArea: Governing council for the area\n",
    "\n",
    "Lattitude: Self explanitory\n",
    "\n",
    "Longtitude: Self explanitory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13580, 21)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melb = pd.read_csv('Data/EDA/melb_data.csv')\n",
    "melb.head()\n",
    "melb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rooms</th>\n",
       "      <th>Price</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Postcode</th>\n",
       "      <th>Bedroom2</th>\n",
       "      <th>Bathroom</th>\n",
       "      <th>Car</th>\n",
       "      <th>Landsize</th>\n",
       "      <th>BuildingArea</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>Lattitude</th>\n",
       "      <th>Longtitude</th>\n",
       "      <th>Propertycount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>13580.000000</td>\n",
       "      <td>1.358000e+04</td>\n",
       "      <td>13580.000000</td>\n",
       "      <td>13580.000000</td>\n",
       "      <td>13580.000000</td>\n",
       "      <td>13580.000000</td>\n",
       "      <td>13518.000000</td>\n",
       "      <td>13580.000000</td>\n",
       "      <td>7130.000000</td>\n",
       "      <td>8205.000000</td>\n",
       "      <td>13580.000000</td>\n",
       "      <td>13580.000000</td>\n",
       "      <td>13580.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.937997</td>\n",
       "      <td>1.075684e+06</td>\n",
       "      <td>10.137776</td>\n",
       "      <td>3105.301915</td>\n",
       "      <td>2.914728</td>\n",
       "      <td>1.534242</td>\n",
       "      <td>1.610075</td>\n",
       "      <td>558.416127</td>\n",
       "      <td>151.967650</td>\n",
       "      <td>1964.684217</td>\n",
       "      <td>-37.809203</td>\n",
       "      <td>144.995216</td>\n",
       "      <td>7454.417378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.955748</td>\n",
       "      <td>6.393107e+05</td>\n",
       "      <td>5.868725</td>\n",
       "      <td>90.676964</td>\n",
       "      <td>0.965921</td>\n",
       "      <td>0.691712</td>\n",
       "      <td>0.962634</td>\n",
       "      <td>3990.669241</td>\n",
       "      <td>541.014538</td>\n",
       "      <td>37.273762</td>\n",
       "      <td>0.079260</td>\n",
       "      <td>0.103916</td>\n",
       "      <td>4378.581772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.500000e+04</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3000.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1196.000000</td>\n",
       "      <td>-38.182550</td>\n",
       "      <td>144.431810</td>\n",
       "      <td>249.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.500000e+05</td>\n",
       "      <td>6.100000</td>\n",
       "      <td>3044.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>177.000000</td>\n",
       "      <td>93.000000</td>\n",
       "      <td>1940.000000</td>\n",
       "      <td>-37.856822</td>\n",
       "      <td>144.929600</td>\n",
       "      <td>4380.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>9.030000e+05</td>\n",
       "      <td>9.200000</td>\n",
       "      <td>3084.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>440.000000</td>\n",
       "      <td>126.000000</td>\n",
       "      <td>1970.000000</td>\n",
       "      <td>-37.802355</td>\n",
       "      <td>145.000100</td>\n",
       "      <td>6555.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.330000e+06</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>3148.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>651.000000</td>\n",
       "      <td>174.000000</td>\n",
       "      <td>1999.000000</td>\n",
       "      <td>-37.756400</td>\n",
       "      <td>145.058305</td>\n",
       "      <td>10331.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>9.000000e+06</td>\n",
       "      <td>48.100000</td>\n",
       "      <td>3977.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>433014.000000</td>\n",
       "      <td>44515.000000</td>\n",
       "      <td>2018.000000</td>\n",
       "      <td>-37.408530</td>\n",
       "      <td>145.526350</td>\n",
       "      <td>21650.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Rooms         Price      Distance      Postcode      Bedroom2  \\\n",
       "count  13580.000000  1.358000e+04  13580.000000  13580.000000  13580.000000   \n",
       "mean       2.937997  1.075684e+06     10.137776   3105.301915      2.914728   \n",
       "std        0.955748  6.393107e+05      5.868725     90.676964      0.965921   \n",
       "min        1.000000  8.500000e+04      0.000000   3000.000000      0.000000   \n",
       "25%        2.000000  6.500000e+05      6.100000   3044.000000      2.000000   \n",
       "50%        3.000000  9.030000e+05      9.200000   3084.000000      3.000000   \n",
       "75%        3.000000  1.330000e+06     13.000000   3148.000000      3.000000   \n",
       "max       10.000000  9.000000e+06     48.100000   3977.000000     20.000000   \n",
       "\n",
       "           Bathroom           Car       Landsize  BuildingArea    YearBuilt  \\\n",
       "count  13580.000000  13518.000000   13580.000000   7130.000000  8205.000000   \n",
       "mean       1.534242      1.610075     558.416127    151.967650  1964.684217   \n",
       "std        0.691712      0.962634    3990.669241    541.014538    37.273762   \n",
       "min        0.000000      0.000000       0.000000      0.000000  1196.000000   \n",
       "25%        1.000000      1.000000     177.000000     93.000000  1940.000000   \n",
       "50%        1.000000      2.000000     440.000000    126.000000  1970.000000   \n",
       "75%        2.000000      2.000000     651.000000    174.000000  1999.000000   \n",
       "max        8.000000     10.000000  433014.000000  44515.000000  2018.000000   \n",
       "\n",
       "          Lattitude    Longtitude  Propertycount  \n",
       "count  13580.000000  13580.000000   13580.000000  \n",
       "mean     -37.809203    144.995216    7454.417378  \n",
       "std        0.079260      0.103916    4378.581772  \n",
       "min      -38.182550    144.431810     249.000000  \n",
       "25%      -37.856822    144.929600    4380.000000  \n",
       "50%      -37.802355    145.000100    6555.000000  \n",
       "75%      -37.756400    145.058305   10331.000000  \n",
       "max      -37.408530    145.526350   21650.000000  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melb.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Suburb', 'Address', 'Type', 'Method', 'SellerG', 'Date', 'CouncilArea',\n",
       "       'Regionname'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melb.select_dtypes(['object']).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.isnull(melb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Suburb              0\n",
       "Address             0\n",
       "Rooms               0\n",
       "Type                0\n",
       "Price               0\n",
       "Method              0\n",
       "SellerG             0\n",
       "Date                0\n",
       "Distance            0\n",
       "Postcode            0\n",
       "Bedroom2            0\n",
       "Bathroom            0\n",
       "Car                62\n",
       "Landsize            0\n",
       "BuildingArea     6450\n",
       "YearBuilt        5375\n",
       "CouncilArea      1369\n",
       "Lattitude           0\n",
       "Longtitude          0\n",
       "Regionname          0\n",
       "Propertycount       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melb.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1075684.079455081"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the mean\n",
    "mean = np.mean(melb.Price)\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Reservoir             359\n",
       "Richmond              260\n",
       "Bentleigh East        249\n",
       "Preston               239\n",
       "Brunswick             222\n",
       "Essendon              220\n",
       "South Yarra           202\n",
       "Glen Iris             195\n",
       "Hawthorn              191\n",
       "Coburg                190\n",
       "Northcote             188\n",
       "Brighton              186\n",
       "Kew                   177\n",
       "Balwyn North          171\n",
       "Pascoe Vale           171\n",
       "Yarraville            164\n",
       "St Kilda              162\n",
       "Glenroy               159\n",
       "Port Melbourne        153\n",
       "Moonee Ponds          149\n",
       "Carnegie              146\n",
       "Bentleigh             139\n",
       "Thornbury             139\n",
       "Brighton East         138\n",
       "Newport               137\n",
       "Footscray             132\n",
       "Camberwell            132\n",
       "Elwood                131\n",
       "Ascot Vale            130\n",
       "Hampton               130\n",
       "                     ... \n",
       "Kooyong                 2\n",
       "Hurstbridge             2\n",
       "Beaconsfield            2\n",
       "North Warrandyte        2\n",
       "Deepdene                2\n",
       "Williams Landing        2\n",
       "Keilor Lodge            2\n",
       "Skye                    2\n",
       "Chirnside Park          2\n",
       "Yarra Glen              1\n",
       "Sandhurst               1\n",
       "Beaconsfield Upper      1\n",
       "Whittlesea              1\n",
       "Healesville             1\n",
       "Croydon South           1\n",
       "Riddells Creek          1\n",
       "Upwey                   1\n",
       "New Gisborne            1\n",
       "Emerald                 1\n",
       "Rockbank                1\n",
       "Wonga Park              1\n",
       "Montrose                1\n",
       "Monbulk                 1\n",
       "Brookfield              1\n",
       "Attwood                 1\n",
       "Officer                 1\n",
       "Plumpton                1\n",
       "Bullengarook            1\n",
       "Bacchus Marsh           1\n",
       "Wallan                  1\n",
       "Name: Suburb, Length: 314, dtype: int64"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melb['Suburb'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Suburb\n",
       "Abbotsford            1.060366e+06\n",
       "Aberfeldie            1.277455e+06\n",
       "Airport West          7.566567e+05\n",
       "Albanvale             5.370833e+05\n",
       "Albert Park           1.941355e+06\n",
       "Albion                5.931463e+05\n",
       "Alphington            1.400824e+06\n",
       "Altona                8.348304e+05\n",
       "Altona Meadows        6.717500e+05\n",
       "Altona North          8.008304e+05\n",
       "Ardeer                6.673333e+05\n",
       "Armadale              1.496716e+06\n",
       "Ascot Vale            1.053227e+06\n",
       "Ashburton             1.728725e+06\n",
       "Ashwood               1.154674e+06\n",
       "Aspendale             1.125286e+06\n",
       "Aspendale Gardens     8.852500e+05\n",
       "Attwood               6.400000e+05\n",
       "Avondale Heights      8.059932e+05\n",
       "Bacchus Marsh         2.850000e+05\n",
       "Balaclava             7.488333e+05\n",
       "Balwyn                1.869879e+06\n",
       "Balwyn North          1.793405e+06\n",
       "Bayswater             7.596429e+05\n",
       "Bayswater North       7.065000e+05\n",
       "Beaconsfield          6.035000e+05\n",
       "Beaconsfield Upper    6.750000e+05\n",
       "Beaumaris             1.725056e+06\n",
       "Bellfield             8.225312e+05\n",
       "Bentleigh             1.279500e+06\n",
       "                          ...     \n",
       "Travancore            1.010450e+06\n",
       "Truganina             4.628700e+05\n",
       "Tullamarine           6.105000e+05\n",
       "Upwey                 5.750000e+05\n",
       "Vermont               1.084000e+06\n",
       "Vermont South         1.228692e+06\n",
       "Viewbank              9.335915e+05\n",
       "Wallan                3.660000e+05\n",
       "Wantirna              9.931667e+05\n",
       "Wantirna South        1.102864e+06\n",
       "Warrandyte            1.219200e+06\n",
       "Waterways             1.248500e+06\n",
       "Watsonia              7.487558e+05\n",
       "Watsonia North        7.635000e+05\n",
       "Werribee              5.107540e+05\n",
       "West Footscray        7.863000e+05\n",
       "West Melbourne        9.945161e+05\n",
       "Westmeadows           6.218571e+05\n",
       "Wheelers Hill         1.251488e+06\n",
       "Whittlesea            6.010000e+05\n",
       "Williams Landing      5.325000e+05\n",
       "Williamstown          1.312481e+06\n",
       "Williamstown North    8.708684e+05\n",
       "Windsor               9.721167e+05\n",
       "Wollert               5.018111e+05\n",
       "Wonga Park            9.000000e+05\n",
       "Wyndham Vale          4.832500e+05\n",
       "Yallambie             8.396875e+05\n",
       "Yarra Glen            6.200000e+05\n",
       "Yarraville            9.886155e+05\n",
       "Name: Price, Length: 314, dtype: float64"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melb.groupby(['Suburb'])['Price'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop rows with missing values\n",
    "melb.dropna(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop columns with missing values \n",
    "melb.dropna(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "obj_cats = ['Suburb', 'Address', 'Type', 'Method', 'SellerG', 'CouncilArea','Regionname']\n",
    "\n",
    "for colname in obj_cats:\n",
    "    melb[colname] = melb[colname].astype('category') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 13580 entries, 0 to 13579\n",
      "Data columns (total 21 columns):\n",
      "Suburb           13580 non-null category\n",
      "Address          13580 non-null category\n",
      "Rooms            13580 non-null int64\n",
      "Type             13580 non-null category\n",
      "Price            13580 non-null float64\n",
      "Method           13580 non-null category\n",
      "SellerG          13580 non-null category\n",
      "Date             13580 non-null object\n",
      "Distance         13580 non-null float64\n",
      "Postcode         13580 non-null float64\n",
      "Bedroom2         13580 non-null float64\n",
      "Bathroom         13580 non-null float64\n",
      "Car              13518 non-null float64\n",
      "Landsize         13580 non-null float64\n",
      "BuildingArea     7130 non-null float64\n",
      "YearBuilt        8205 non-null float64\n",
      "CouncilArea      12211 non-null category\n",
      "Lattitude        13580 non-null float64\n",
      "Longtitude       13580 non-null float64\n",
      "Regionname       13580 non-null category\n",
      "Propertycount    13580 non-null float64\n",
      "dtypes: category(7), float64(12), int64(1), object(1)\n",
      "memory usage: 2.3+ MB\n"
     ]
    }
   ],
   "source": [
    "melb.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Rooms</th>\n",
       "      <td>13580.0</td>\n",
       "      <td>2.937997e+00</td>\n",
       "      <td>0.955748</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Price</th>\n",
       "      <td>13580.0</td>\n",
       "      <td>1.075684e+06</td>\n",
       "      <td>639310.724296</td>\n",
       "      <td>85000.00000</td>\n",
       "      <td>650000.000000</td>\n",
       "      <td>903000.000000</td>\n",
       "      <td>1.330000e+06</td>\n",
       "      <td>9.000000e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Distance</th>\n",
       "      <td>13580.0</td>\n",
       "      <td>1.013778e+01</td>\n",
       "      <td>5.868725</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>6.100000</td>\n",
       "      <td>9.200000</td>\n",
       "      <td>1.300000e+01</td>\n",
       "      <td>4.810000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Postcode</th>\n",
       "      <td>13580.0</td>\n",
       "      <td>3.105302e+03</td>\n",
       "      <td>90.676964</td>\n",
       "      <td>3000.00000</td>\n",
       "      <td>3044.000000</td>\n",
       "      <td>3084.000000</td>\n",
       "      <td>3.148000e+03</td>\n",
       "      <td>3.977000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bedroom2</th>\n",
       "      <td>13580.0</td>\n",
       "      <td>2.914728e+00</td>\n",
       "      <td>0.965921</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>2.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bathroom</th>\n",
       "      <td>13580.0</td>\n",
       "      <td>1.534242e+00</td>\n",
       "      <td>0.691712</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>8.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Car</th>\n",
       "      <td>13518.0</td>\n",
       "      <td>1.610075e+00</td>\n",
       "      <td>0.962634</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Landsize</th>\n",
       "      <td>13580.0</td>\n",
       "      <td>5.584161e+02</td>\n",
       "      <td>3990.669241</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>177.000000</td>\n",
       "      <td>440.000000</td>\n",
       "      <td>6.510000e+02</td>\n",
       "      <td>4.330140e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BuildingArea</th>\n",
       "      <td>7130.0</td>\n",
       "      <td>1.519676e+02</td>\n",
       "      <td>541.014538</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>93.000000</td>\n",
       "      <td>126.000000</td>\n",
       "      <td>1.740000e+02</td>\n",
       "      <td>4.451500e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>YearBuilt</th>\n",
       "      <td>8205.0</td>\n",
       "      <td>1.964684e+03</td>\n",
       "      <td>37.273762</td>\n",
       "      <td>1196.00000</td>\n",
       "      <td>1940.000000</td>\n",
       "      <td>1970.000000</td>\n",
       "      <td>1.999000e+03</td>\n",
       "      <td>2.018000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lattitude</th>\n",
       "      <td>13580.0</td>\n",
       "      <td>-3.780920e+01</td>\n",
       "      <td>0.079260</td>\n",
       "      <td>-38.18255</td>\n",
       "      <td>-37.856822</td>\n",
       "      <td>-37.802355</td>\n",
       "      <td>-3.775640e+01</td>\n",
       "      <td>-3.740853e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Longtitude</th>\n",
       "      <td>13580.0</td>\n",
       "      <td>1.449952e+02</td>\n",
       "      <td>0.103916</td>\n",
       "      <td>144.43181</td>\n",
       "      <td>144.929600</td>\n",
       "      <td>145.000100</td>\n",
       "      <td>1.450583e+02</td>\n",
       "      <td>1.455264e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Propertycount</th>\n",
       "      <td>13580.0</td>\n",
       "      <td>7.454417e+03</td>\n",
       "      <td>4378.581772</td>\n",
       "      <td>249.00000</td>\n",
       "      <td>4380.000000</td>\n",
       "      <td>6555.000000</td>\n",
       "      <td>1.033100e+04</td>\n",
       "      <td>2.165000e+04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 count          mean            std          min  \\\n",
       "Rooms          13580.0  2.937997e+00       0.955748      1.00000   \n",
       "Price          13580.0  1.075684e+06  639310.724296  85000.00000   \n",
       "Distance       13580.0  1.013778e+01       5.868725      0.00000   \n",
       "Postcode       13580.0  3.105302e+03      90.676964   3000.00000   \n",
       "Bedroom2       13580.0  2.914728e+00       0.965921      0.00000   \n",
       "Bathroom       13580.0  1.534242e+00       0.691712      0.00000   \n",
       "Car            13518.0  1.610075e+00       0.962634      0.00000   \n",
       "Landsize       13580.0  5.584161e+02    3990.669241      0.00000   \n",
       "BuildingArea    7130.0  1.519676e+02     541.014538      0.00000   \n",
       "YearBuilt       8205.0  1.964684e+03      37.273762   1196.00000   \n",
       "Lattitude      13580.0 -3.780920e+01       0.079260    -38.18255   \n",
       "Longtitude     13580.0  1.449952e+02       0.103916    144.43181   \n",
       "Propertycount  13580.0  7.454417e+03    4378.581772    249.00000   \n",
       "\n",
       "                         25%            50%           75%           max  \n",
       "Rooms               2.000000       3.000000  3.000000e+00  1.000000e+01  \n",
       "Price          650000.000000  903000.000000  1.330000e+06  9.000000e+06  \n",
       "Distance            6.100000       9.200000  1.300000e+01  4.810000e+01  \n",
       "Postcode         3044.000000    3084.000000  3.148000e+03  3.977000e+03  \n",
       "Bedroom2            2.000000       3.000000  3.000000e+00  2.000000e+01  \n",
       "Bathroom            1.000000       1.000000  2.000000e+00  8.000000e+00  \n",
       "Car                 1.000000       2.000000  2.000000e+00  1.000000e+01  \n",
       "Landsize          177.000000     440.000000  6.510000e+02  4.330140e+05  \n",
       "BuildingArea       93.000000     126.000000  1.740000e+02  4.451500e+04  \n",
       "YearBuilt        1940.000000    1970.000000  1.999000e+03  2.018000e+03  \n",
       "Lattitude         -37.856822     -37.802355 -3.775640e+01 -3.740853e+01  \n",
       "Longtitude        144.929600     145.000100  1.450583e+02  1.455264e+02  \n",
       "Propertycount    4380.000000    6555.000000  1.033100e+04  2.165000e+04  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melb.describe().transpose()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interpolation\n",
    "\n",
    "Alternatively, you can also choose to interpolate missing values: the interpolate() function will perform a linear interpolation at the missing data points to “guess” the value that is most likely to be filled in.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "melb_interpolated = melb.interpolate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Suburb              0\n",
       "Address             0\n",
       "Rooms               0\n",
       "Type                0\n",
       "Price               0\n",
       "Method              0\n",
       "SellerG             0\n",
       "Date                0\n",
       "Distance            0\n",
       "Postcode            0\n",
       "Bedroom2            0\n",
       "Bathroom            0\n",
       "Car                 0\n",
       "Landsize            0\n",
       "BuildingArea        1\n",
       "YearBuilt           1\n",
       "CouncilArea      1369\n",
       "Lattitude           0\n",
       "Longtitude          0\n",
       "Regionname          0\n",
       "Propertycount       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melb_interpolated.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Querying and indexing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>SepalLengthCm</th>\n",
       "      <th>SepalWidthCm</th>\n",
       "      <th>PetalLengthCm</th>\n",
       "      <th>PetalWidthCm</th>\n",
       "      <th>Species</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Id, SepalLengthCm, SepalWidthCm, PetalLengthCm, PetalWidthCm, Species]\n",
       "Index: []"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Petal length greater than sepal length?\n",
    "iris.query('PetalLengthCm > SepalLengthCm')\n",
    "\n",
    "# Petal length equals sepal length?\n",
    "iris.query('PetalLengthCm == SepalLengthCm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Suburb</th>\n",
       "      <th>Address</th>\n",
       "      <th>Rooms</th>\n",
       "      <th>Type</th>\n",
       "      <th>Price</th>\n",
       "      <th>Method</th>\n",
       "      <th>SellerG</th>\n",
       "      <th>Date</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Postcode</th>\n",
       "      <th>...</th>\n",
       "      <th>Bathroom</th>\n",
       "      <th>Car</th>\n",
       "      <th>Landsize</th>\n",
       "      <th>BuildingArea</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>CouncilArea</th>\n",
       "      <th>Lattitude</th>\n",
       "      <th>Longtitude</th>\n",
       "      <th>Regionname</th>\n",
       "      <th>Propertycount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7692</th>\n",
       "      <td>Canterbury</td>\n",
       "      <td>49 Mangarra Rd</td>\n",
       "      <td>5</td>\n",
       "      <td>h</td>\n",
       "      <td>8000000.0</td>\n",
       "      <td>VB</td>\n",
       "      <td>Sotheby's</td>\n",
       "      <td>13/05/2017</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3126.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2079.0</td>\n",
       "      <td>464.3</td>\n",
       "      <td>1880.0</td>\n",
       "      <td>Boroondara</td>\n",
       "      <td>-37.81790</td>\n",
       "      <td>145.06940</td>\n",
       "      <td>Southern Metropolitan</td>\n",
       "      <td>3265.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9575</th>\n",
       "      <td>Hawthorn</td>\n",
       "      <td>49 Lisson Gr</td>\n",
       "      <td>4</td>\n",
       "      <td>h</td>\n",
       "      <td>7650000.0</td>\n",
       "      <td>S</td>\n",
       "      <td>Abercromby's</td>\n",
       "      <td>17/06/2017</td>\n",
       "      <td>5.3</td>\n",
       "      <td>3122.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1690.0</td>\n",
       "      <td>284.0</td>\n",
       "      <td>1863.0</td>\n",
       "      <td>Boroondara</td>\n",
       "      <td>-37.82652</td>\n",
       "      <td>145.03052</td>\n",
       "      <td>Southern Metropolitan</td>\n",
       "      <td>11308.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12094</th>\n",
       "      <td>Mulgrave</td>\n",
       "      <td>35 Bevis St</td>\n",
       "      <td>3</td>\n",
       "      <td>h</td>\n",
       "      <td>9000000.0</td>\n",
       "      <td>PI</td>\n",
       "      <td>Hall</td>\n",
       "      <td>29/07/2017</td>\n",
       "      <td>18.8</td>\n",
       "      <td>3170.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>744.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>1960.0</td>\n",
       "      <td>Monash</td>\n",
       "      <td>-37.93168</td>\n",
       "      <td>145.16126</td>\n",
       "      <td>South-Eastern Metropolitan</td>\n",
       "      <td>7113.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Suburb         Address  Rooms Type      Price Method       SellerG  \\\n",
       "7692   Canterbury  49 Mangarra Rd      5    h  8000000.0     VB     Sotheby's   \n",
       "9575     Hawthorn    49 Lisson Gr      4    h  7650000.0      S  Abercromby's   \n",
       "12094    Mulgrave     35 Bevis St      3    h  9000000.0     PI          Hall   \n",
       "\n",
       "             Date  Distance  Postcode      ...       Bathroom  Car  Landsize  \\\n",
       "7692   13/05/2017       9.0    3126.0      ...            5.0  4.0    2079.0   \n",
       "9575   17/06/2017       5.3    3122.0      ...            2.0  4.0    1690.0   \n",
       "12094  29/07/2017      18.8    3170.0      ...            1.0  1.0     744.0   \n",
       "\n",
       "       BuildingArea  YearBuilt  CouncilArea Lattitude  Longtitude  \\\n",
       "7692          464.3     1880.0   Boroondara -37.81790   145.06940   \n",
       "9575          284.0     1863.0   Boroondara -37.82652   145.03052   \n",
       "12094         117.0     1960.0       Monash -37.93168   145.16126   \n",
       "\n",
       "                       Regionname Propertycount  \n",
       "7692        Southern Metropolitan        3265.0  \n",
       "9575        Southern Metropolitan       11308.0  \n",
       "12094  South-Eastern Metropolitan        7113.0  \n",
       "\n",
       "[3 rows x 21 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melb.query('Price >7000000')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Correlation\n",
    "\n",
    "In statistics, correlation or dependence is any statistical relationship, whether causal or not, between two random variables or bivariate data. In the broadest sense correlation is any statistical association, though it commonly refers to the degree to which a pair of variables are linearly related"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>SepalLengthCm</th>\n",
       "      <th>SepalWidthCm</th>\n",
       "      <th>PetalLengthCm</th>\n",
       "      <th>PetalWidthCm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Id</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.734054</td>\n",
       "      <td>-0.411539</td>\n",
       "      <td>0.868498</td>\n",
       "      <td>0.879045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SepalLengthCm</th>\n",
       "      <td>0.734054</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.159457</td>\n",
       "      <td>0.881386</td>\n",
       "      <td>0.834421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SepalWidthCm</th>\n",
       "      <td>-0.411539</td>\n",
       "      <td>-0.159457</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.303421</td>\n",
       "      <td>-0.277511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PetalLengthCm</th>\n",
       "      <td>0.868498</td>\n",
       "      <td>0.881386</td>\n",
       "      <td>-0.303421</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.936003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PetalWidthCm</th>\n",
       "      <td>0.879045</td>\n",
       "      <td>0.834421</td>\n",
       "      <td>-0.277511</td>\n",
       "      <td>0.936003</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Id  SepalLengthCm  SepalWidthCm  PetalLengthCm  \\\n",
       "Id             1.000000       0.734054     -0.411539       0.868498   \n",
       "SepalLengthCm  0.734054       1.000000     -0.159457       0.881386   \n",
       "SepalWidthCm  -0.411539      -0.159457      1.000000      -0.303421   \n",
       "PetalLengthCm  0.868498       0.881386     -0.303421       1.000000   \n",
       "PetalWidthCm   0.879045       0.834421     -0.277511       0.936003   \n",
       "\n",
       "               PetalWidthCm  \n",
       "Id                 0.879045  \n",
       "SepalLengthCm      0.834421  \n",
       "SepalWidthCm      -0.277511  \n",
       "PetalLengthCm      0.936003  \n",
       "PetalWidthCm       1.000000  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pearson correlation\n",
    "iris.corr()\n",
    "\n",
    "# Kendall Tau correlation\n",
    "iris.corr('kendall')\n",
    "\n",
    "# Spearman Rank correlation\n",
    "iris.corr('spearman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}