PythonForDataAnalysis/pfda_ch12.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "language_info": {
   "name": "python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   }
  },
  "orig_nbformat": 2,
  "file_extension": ".py",
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "cells": [
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Chapter 12. Advanced pandas\n",
    "\n",
    "## 12.1 Categorical data\n",
    "\n",
    "This section introduces the pandas `Categorical` type.\n",
    "It can often be more performance and memory efficient than the string equivalent.\n",
    "\n",
    "### Background and motivation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0     apple\n1    orange\n2     apple\n3     apple\n4     apple\n5    orange\n6     apple\n7     apple\ndtype: object"
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values = pd.Series(['apple', 'orange', 'apple', 'apple'] * 2)\n",
    "values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array(['apple', 'orange'], dtype=object)"
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.unique(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "apple     6\norange    2\ndtype: int64"
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(values)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "A good solution for sotring categorical data is to use a dimension table."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0    0\n1    1\n2    0\n3    0\n4    0\n5    1\n6    0\n7    0\ndtype: int64"
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values = pd.Series([0, 1, 0, 0] * 2)\n",
    "dim = pd.Series(['apple', 'orange'])\n",
    "values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0     apple\n1    orange\n0     apple\n0     apple\n0     apple\n1    orange\n0     apple\n0     apple\ndtype: object"
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim[values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0     apple\n1    orange\n0     apple\n0     apple\n0     apple\n1    orange\n0     apple\n0     apple\ndtype: object"
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim.take(values)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Categorical type in pandas\n",
    "\n",
    "The `Categorical` type uses an integer-based encoding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>basket_id</th>\n      <th>fruit</th>\n      <th>count</th>\n      <th>weight</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>apple</td>\n      <td>8</td>\n      <td>2.583576</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>orange</td>\n      <td>3</td>\n      <td>1.750349</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>apple</td>\n      <td>6</td>\n      <td>3.567092</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>apple</td>\n      <td>14</td>\n      <td>3.854651</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>apple</td>\n      <td>6</td>\n      <td>1.533766</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>5</td>\n      <td>orange</td>\n      <td>10</td>\n      <td>3.166900</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>6</td>\n      <td>apple</td>\n      <td>12</td>\n      <td>2.115580</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>7</td>\n      <td>apple</td>\n      <td>6</td>\n      <td>2.272178</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "   basket_id   fruit  count    weight\n0          0   apple      8  2.583576\n1          1  orange      3  1.750349\n2          2   apple      6  3.567092\n3          3   apple     14  3.854651\n4          4   apple      6  1.533766\n5          5  orange     10  3.166900\n6          6   apple     12  2.115580\n7          7   apple      6  2.272178"
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fruits = ['apple', 'orange', 'apple', 'apple'] * 2\n",
    "N = len(fruits)\n",
    "df = pd.DataFrame({\n",
    "        'fruit': fruits,\n",
    "        'basket_id': np.arange(N),\n",
    "        'count': np.random.randint(3, 15, size=N),\n",
    "        'weight': np.random.uniform(0, 4, size=N)\n",
    "    }, columns = ['basket_id', 'fruit', 'count', 'weight'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0     apple\n1    orange\n2     apple\n3     apple\n4     apple\n5    orange\n6     apple\n7     apple\nName: fruit, dtype: category\nCategories (2, object): [apple, orange]"
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Turn the fruit data into a Categorical\n",
    "fruit_cat = df['fruit'].astype('category')\n",
    "fruit_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "pandas.core.arrays.categorical.Categorical"
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = fruit_cat.values\n",
    "type(c)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "This Categorical object has `categories` and `codes` attributes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "Index(['apple', 'orange'], dtype='object')"
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array([0, 1, 0, 0, 0, 1, 0, 0], dtype=int8)"
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>basket_id</th>\n      <th>fruit</th>\n      <th>count</th>\n      <th>weight</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>apple</td>\n      <td>8</td>\n      <td>2.583576</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>orange</td>\n      <td>3</td>\n      <td>1.750349</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>apple</td>\n      <td>6</td>\n      <td>3.567092</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>apple</td>\n      <td>14</td>\n      <td>3.854651</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>apple</td>\n      <td>6</td>\n      <td>1.533766</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>5</td>\n      <td>orange</td>\n      <td>10</td>\n      <td>3.166900</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>6</td>\n      <td>apple</td>\n      <td>12</td>\n      <td>2.115580</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>7</td>\n      <td>apple</td>\n      <td>6</td>\n      <td>2.272178</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "   basket_id   fruit  count    weight\n0          0   apple      8  2.583576\n1          1  orange      3  1.750349\n2          2   apple      6  3.567092\n3          3   apple     14  3.854651\n4          4   apple      6  1.533766\n5          5  orange     10  3.166900\n6          6   apple     12  2.115580\n7          7   apple      6  2.272178"
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert the DataFrame column into a Categorical\n",
    "df['fruit'] = df['fruit'].astype('category')\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "A Categorical object can be created from the codes and categories using the `from_codes()` method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "[foo, bar, baz, foo, foo, bar]\nCategories (3, object): [foo, bar, baz]"
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categories = ['foo', 'bar', 'baz']\n",
    "codes = [0, 1, 2, 0, 0, 1]\n",
    "my_cats_2 = pd.Categorical.from_codes(codes, categories)\n",
    "my_cats_2"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "By default, no order is assumed for the data, though it can be by setting `ordered=True`.\n",
    "It can also be removed with the `as_unordered()` method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "[foo, bar, baz, foo, foo, bar]\nCategories (3, object): [foo < bar < baz]"
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordered_cat = pd.Categorical.from_codes(codes, categories, ordered=True)\n",
    "ordered_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "[foo, bar, baz, foo, foo, bar]\nCategories (3, object): [foo, bar, baz]"
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordered_cat.as_unordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_cats_2.as_ordered(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "[foo, bar, baz, foo, foo, bar]\nCategories (3, object): [foo < bar < baz]"
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cats_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "True"
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cats_2.ordered"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Computations with categoricals\n",
    "\n",
    "Most uses of Categorical will behave as if the data were still an unencoded structure (such as an array of strings).\n",
    "Some pandas operations, such as `groupby()`, and other functions take advantage of the coded nature of Categorical for performance enhancements.\n",
    "\n",
    "the `qcut()` and `cut()` pandas functions return Categoricals."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array([ 0.48431215,  0.57914048, -0.18158257,  1.41020463, -0.37447169])"
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draws = np.random.randn(1000)\n",
    "draws[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "[(-0.0301, 0.599], (-0.0301, 0.599], (-0.741, -0.0301], (0.599, 3.102], (-0.741, -0.0301], ..., (-0.0301, 0.599], (0.599, 3.102], (-0.741, -0.0301], (-0.741, -0.0301], (-0.0301, 0.599]]\nLength: 1000\nCategories (4, interval[float64]): [(-3.516, -0.741] < (-0.741, -0.0301] < (-0.0301, 0.599] < (0.599, 3.102]]"
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins = pd.qcut(draws, 4)\n",
    "bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "[Q3, Q3, Q2, Q4, Q2, ..., Q3, Q4, Q2, Q2, Q3]\nLength: 1000\nCategories (4, object): [Q1 < Q2 < Q3 < Q4]"
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The same quartiles but with more helpful names.\n",
    "bins = pd.qcut(draws, 4, labels=['Q1', 'Q2', 'Q3', 'Q4'])\n",
    "bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>quartile</th>\n      <th>count</th>\n      <th>min</th>\n      <th>max</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Q1</td>\n      <td>250</td>\n      <td>-3.514768</td>\n      <td>-0.742269</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Q2</td>\n      <td>250</td>\n      <td>-0.740747</td>\n      <td>-0.031245</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Q3</td>\n      <td>250</td>\n      <td>-0.028946</td>\n      <td>0.596688</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Q4</td>\n      <td>250</td>\n      <td>0.604160</td>\n      <td>3.101531</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "  quartile  count       min       max\n0       Q1    250 -3.514768 -0.742269\n1       Q2    250 -0.740747 -0.031245\n2       Q3    250 -0.028946  0.596688\n3       Q4    250  0.604160  3.101531"
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins = pd.Series(bins, name='quartile')\n",
    "results = (\n",
    "    pd.Series(draws)\n",
    "        .groupby(bins)\n",
    "        .agg(['count', 'min', 'max'])\n",
    "        .reset_index()\n",
    ")\n",
    "\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0    Q1\n1    Q2\n2    Q3\n3    Q4\nName: quartile, dtype: category\nCategories (4, object): [Q1 < Q2 < Q3 < Q4]"
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.quartile"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Using a Categorical in a DataFrame will provide improvements in speed of computations and resource-consumption by the DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 1000000\n",
    "\n",
    "draws = pd.Series(np.random.randn(N))\n",
    "\n",
    "labels = pd.Series(['foo', 'bar', 'baz', 'qux'] * (N // 4))\n",
    "\n",
    "categories = labels.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "8000128"
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.memory_usage()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "1000320"
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categories.memory_usage()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Categorical methods\n",
    "\n",
    "There are a few extra conviencne methods provided for Categorical objects.\n",
    "They are accessed via the `cat` attribute of a Categorical."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0    a\n1    b\n2    c\n3    d\n4    a\n5    b\n6    c\n7    d\ndtype: category\nCategories (4, object): [a, b, c, d]"
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = pd.Series(['a', 'b', 'c', 'd'] * 2)\n",
    "cat_s = s.astype('category')\n",
    "cat_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0    0\n1    1\n2    2\n3    3\n4    0\n5    1\n6    2\n7    3\ndtype: int8"
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s.cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "Index(['a', 'b', 'c', 'd'], dtype='object')"
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s.cat.categories"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "The categories can be changed, even extended beyond the actual values used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0    a\n1    b\n2    c\n3    d\n4    a\n5    b\n6    c\n7    d\ndtype: category\nCategories (5, object): [a, b, c, d, e]"
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "actual_categories = ['a', 'b', 'c', 'd', 'e']\n",
    "cat_s2 = cat_s.cat.set_categories(actual_categories)\n",
    "cat_s2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "d    2\nc    2\nb    2\na    2\ndtype: int64"
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "d    2\nc    2\nb    2\na    2\ne    0\ndtype: int64"
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s2.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Unused categories can be removed with the `remove_unused_categories()` method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0    a\n1    b\n4    a\n5    b\ndtype: category\nCategories (4, object): [a, b, c, d]"
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s3 = cat_s[cat_s.isin(['a', 'b'])]\n",
    "cat_s3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0    a\n1    b\n4    a\n5    b\ndtype: category\nCategories (2, object): [a, b]"
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s3.cat.remove_unused_categories()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Once final example use-case for Categorical is to create dummy variables for modeling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>a</th>\n      <th>b</th>\n      <th>c</th>\n      <th>d</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "   a  b  c  d\n0  1  0  0  0\n1  0  1  0  0\n2  0  0  1  0\n3  0  0  0  1\n4  1  0  0  0\n5  0  1  0  0\n6  0  0  1  0\n7  0  0  0  1"
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s = pd.Series(['a', 'b', 'c', 'd'] * 2, dtype='category')\n",
    "pd.get_dummies(cat_s)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 12.2 Advanced groupby use\n",
    "\n",
    "### Group transformations and \"unwrapped\" groupbys\n",
    "\n",
    "As discussed earlier, the `apply()` method in grouped operations perform a transformation on the group, one at a time.\n",
    "There is a similar method called `transform()` that has the following constraints on the function it can use:\n",
    "\n",
    "- it can produce a scalar value\n",
    "- it can produce an object of the same shape as the input\n",
    "- it must *not* mutate the input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>key</th>\n      <th>value</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>a</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>b</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>c</td>\n      <td>2.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>a</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>b</td>\n      <td>4.0</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>c</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>a</td>\n      <td>6.0</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>b</td>\n      <td>7.0</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>c</td>\n      <td>8.0</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>a</td>\n      <td>9.0</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>b</td>\n      <td>10.0</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>c</td>\n      <td>11.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "   key  value\n0    a    0.0\n1    b    1.0\n2    c    2.0\n3    a    3.0\n4    b    4.0\n5    c    5.0\n6    a    6.0\n7    b    7.0\n8    c    8.0\n9    a    9.0\n10   b   10.0\n11   c   11.0"
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'key': ['a', 'b', 'c'] * 4,\n",
    "                   'value': np.arange(12.0)})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "key\na    4.5\nb    5.5\nc    6.5\nName: value, dtype: float64"
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = df.groupby('key').value\n",
    "g.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "The following transformation returns a Series of the same length as the input with the means of each group."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0     4.5\n1     5.5\n2     6.5\n3     4.5\n4     5.5\n5     6.5\n6     4.5\n7     5.5\n8     6.5\n9     4.5\n10    5.5\n11    6.5\nName: value, dtype: float64"
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g.transform(lambda x: x.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "0     4.5\n1     5.5\n2     6.5\n3     4.5\n4     5.5\n5     6.5\n6     4.5\n7     5.5\n8     6.5\n9     4.5\n10    5.5\n11    6.5\nName: value, dtype: float64"
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We can call `mean` by name because it is a built-in aggregation method.\n",
    "g.transform('mean')"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 12.3 Techniques for method chaining\n",
    "\n",
    "The `assign()` method is useful for adding columns in method chains.\n",
    "It does *not* modify in-place, but instead returns a new DataFrame.\n",
    "The following two statements are equivalent:\n",
    "\n",
    "```python\n",
    "df2 = df.copy()\n",
    "df2['k'] = v\n",
    "\n",
    "df2 = df.assign(k=v)\n",
    "```\n",
    "\n",
    "Another useful tool for functional programming in Python is the *callable*.\n",
    "Essentially, the entire DataFrame can be passed as a parameter to the supplied function; here is an example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>a</th>\n      <th>b</th>\n      <th>c</th>\n      <th>d</th>\n      <th>message</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n      <td>4</td>\n      <td>hello</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n      <td>8</td>\n      <td>world</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n      <td>12</td>\n      <td>foo</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "   a   b   c   d message\n0  1   2   3   4   hello\n1  5   6   7   8   world\n2  9  10  11  12     foo"
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(\"assets/examples/ex1.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>a</th>\n      <th>b</th>\n      <th>c</th>\n      <th>d</th>\n      <th>message</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n      <td>4</td>\n      <td>hello</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n      <td>8</td>\n      <td>world</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "   a  b  c  d message\n0  1  2  3  4   hello\n1  5  6  7  8   world"
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(pd.read_csv(\"assets/examples/ex1.csv\")\n",
    "    [lambda x: x.b < 7])"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### The pipe method\n",
    "\n",
    "The `pipe()` method is useful for chaining functions that can pass arguments to each other sequentially.\n",
    "\n",
    "```python\n",
    "a = function1(df, arg1=v1)\n",
    "b = function2(a, arg3=v3)\n",
    "c = function3(b, arg4=v4)\n",
    "```\n",
    "\n",
    "```python\n",
    "c = (df.pipe(function1, arg1=v1)\n",
    "    .pipe(function2, arg3=v3)\n",
    "    .pipe(function3, arg4=v4))\n",
    "```"
   ]
  }
 ]
}