PythonForDataAnalysis/pfda_ch07.ipynb

{"cells":[{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["# Chapter 7. Data Cleaning and Preparation\n","\n","In this chapter, the author reviewed tools for missing data, duplicate data, string manipulation, and a few other other common practices used for data preparation.\n","\n","## 7.1 Handling missing data\n","\n","Missing data is ignored by default when computing summary and descriptive statistics on a DataFrame or Series.\n","For numeric data, the floating-point value `NaN` (\"not a number\") is used."]},{"cell_type":"code","execution_count":186,"metadata":{},"outputs":[],"source":["import pandas as pd\n","import numpy as np\n","\n","np.random.seed(0)"]},{"cell_type":"code","execution_count":187,"metadata":{},"outputs":[{"data":{"text/plain":"0     aardvark\n1    artichoke\n2          NaN\n3      avocado\ndtype: object"},"execution_count":187,"metadata":{},"output_type":"execute_result"}],"source":["string_data = pd.Series(['aardvark', 'artichoke', np.nan, 'avocado'])\n","string_data"]},{"cell_type":"code","execution_count":188,"metadata":{},"outputs":[{"data":{"text/plain":"0    False\n1    False\n2     True\n3    False\ndtype: bool"},"execution_count":188,"metadata":{},"output_type":"execute_result"}],"source":["string_data.isnull()"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["### Filtering out missing data\n","\n","The `dropna()` method removes `NaN` values from a Series."]},{"cell_type":"code","execution_count":189,"metadata":{},"outputs":[{"data":{"text/plain":"0    1.0\n1    NaN\n2    3.5\n3    NaN\n4    7.0\ndtype: float64"},"execution_count":189,"metadata":{},"output_type":"execute_result"}],"source":["from numpy import nan as NA\n","\n","data = pd.Series([1, NA, 3.5, NA, 7])\n","data"]},{"cell_type":"code","execution_count":190,"metadata":{},"outputs":[{"data":{"text/plain":"0    1.0\n2    3.5\n4    7.0\ndtype: float64"},"execution_count":190,"metadata":{},"output_type":"execute_result"}],"source":["data.dropna()"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["On a DataFrame, `dropna()` removes rows with an `NaN` values."]},{"cell_type":"code","execution_count":191,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"     0    1    2\n0  1.0  6.5  3.0\n1  1.0  NaN  NaN\n2  NaN  NaN  NaN\n3  NaN  6.5  3.0"},"execution_count":191,"metadata":{},"output_type":"execute_result"}],"source":["data = pd.DataFrame([[1., 6.5, 3.],\n","                     [1., NA, NA],\n","                     [NA, NA, NA],\n","                     [NA, 6.5, 3.]])\n","data"]},{"cell_type":"code","execution_count":192,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"     0    1    2\n0  1.0  6.5  3.0"},"execution_count":192,"metadata":{},"output_type":"execute_result"}],"source":["cleaned = data.dropna()\n","cleaned"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["However, passing `how='all'` will remove only rows with *all* `NaN`."]},{"cell_type":"code","execution_count":193,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"     0    1    2\n0  1.0  6.5  3.0\n1  1.0  NaN  NaN\n3  NaN  6.5  3.0"},"execution_count":193,"metadata":{},"output_type":"execute_result"}],"source":["data.dropna(how='all')"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["Or only columns made up of all `NaN` can be dropped by declaring the axis."]},{"cell_type":"code","execution_count":194,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>4</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"     0    1    2   4\n0  1.0  6.5  3.0 NaN\n1  1.0  NaN  NaN NaN\n2  NaN  NaN  NaN NaN\n3  NaN  6.5  3.0 NaN"},"execution_count":194,"metadata":{},"output_type":"execute_result"}],"source":["data[4] = NA\n","data"]},{"cell_type":"code","execution_count":195,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"     0    1    2\n0  1.0  6.5  3.0\n1  1.0  NaN  NaN\n2  NaN  NaN  NaN\n3  NaN  6.5  3.0"},"execution_count":195,"metadata":{},"output_type":"execute_result"}],"source":["data.dropna(axis=1, how='all')"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["You can also specify the maximum number of missing data values allowed for an individual row."]},{"cell_type":"code","execution_count":196,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.764052</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2.240893</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.950088</td>\n      <td>NaN</td>\n      <td>-0.103219</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.410599</td>\n      <td>NaN</td>\n      <td>1.454274</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.761038</td>\n      <td>0.121675</td>\n      <td>0.443863</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.333674</td>\n      <td>1.494079</td>\n      <td>-0.205158</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.313068</td>\n      <td>-0.854096</td>\n      <td>-2.552990</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          0         1         2\n0  1.764052       NaN       NaN\n1  2.240893       NaN       NaN\n2  0.950088       NaN -0.103219\n3  0.410599       NaN  1.454274\n4  0.761038  0.121675  0.443863\n5  0.333674  1.494079 -0.205158\n6  0.313068 -0.854096 -2.552990"},"execution_count":196,"metadata":{},"output_type":"execute_result"}],"source":["df = pd.DataFrame(np.random.randn(7, 3))\n","df.iloc[:4, 1] = NA\n","df.iloc[:2, 2] = NA\n","df"]},{"cell_type":"code","execution_count":197,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>4</th>\n      <td>0.761038</td>\n      <td>0.121675</td>\n      <td>0.443863</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.333674</td>\n      <td>1.494079</td>\n      <td>-0.205158</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.313068</td>\n      <td>-0.854096</td>\n      <td>-2.552990</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          0         1         2\n4  0.761038  0.121675  0.443863\n5  0.333674  1.494079 -0.205158\n6  0.313068 -0.854096 -2.552990"},"execution_count":197,"metadata":{},"output_type":"execute_result"}],"source":["df.dropna()"]},{"cell_type":"code","execution_count":198,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2</th>\n      <td>0.950088</td>\n      <td>NaN</td>\n      <td>-0.103219</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.410599</td>\n      <td>NaN</td>\n      <td>1.454274</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.761038</td>\n      <td>0.121675</td>\n      <td>0.443863</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.333674</td>\n      <td>1.494079</td>\n      <td>-0.205158</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.313068</td>\n      <td>-0.854096</td>\n      <td>-2.552990</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          0         1         2\n2  0.950088       NaN -0.103219\n3  0.410599       NaN  1.454274\n4  0.761038  0.121675  0.443863\n5  0.333674  1.494079 -0.205158\n6  0.313068 -0.854096 -2.552990"},"execution_count":198,"metadata":{},"output_type":"execute_result"}],"source":["df.dropna(thresh=2)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["### Filling in missing data\n","\n","`fillna()` can be used in most cases to fill in missing data."]},{"cell_type":"code","execution_count":199,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.764052</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2.240893</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.950088</td>\n      <td>0.000000</td>\n      <td>-0.103219</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.410599</td>\n      <td>0.000000</td>\n      <td>1.454274</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.761038</td>\n      <td>0.121675</td>\n      <td>0.443863</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.333674</td>\n      <td>1.494079</td>\n      <td>-0.205158</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.313068</td>\n      <td>-0.854096</td>\n      <td>-2.552990</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          0         1         2\n0  1.764052  0.000000  0.000000\n1  2.240893  0.000000  0.000000\n2  0.950088  0.000000 -0.103219\n3  0.410599  0.000000  1.454274\n4  0.761038  0.121675  0.443863\n5  0.333674  1.494079 -0.205158\n6  0.313068 -0.854096 -2.552990"},"execution_count":199,"metadata":{},"output_type":"execute_result"}],"source":["df.fillna(0)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["A dictionary can be used to fill specific columns with specific values."]},{"cell_type":"code","execution_count":200,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.764052</td>\n      <td>0.500000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2.240893</td>\n      <td>0.500000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.950088</td>\n      <td>0.500000</td>\n      <td>-0.103219</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.410599</td>\n      <td>0.500000</td>\n      <td>1.454274</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.761038</td>\n      <td>0.121675</td>\n      <td>0.443863</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.333674</td>\n      <td>1.494079</td>\n      <td>-0.205158</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.313068</td>\n      <td>-0.854096</td>\n      <td>-2.552990</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          0         1         2\n0  1.764052  0.500000  0.000000\n1  2.240893  0.500000  0.000000\n2  0.950088  0.500000 -0.103219\n3  0.410599  0.500000  1.454274\n4  0.761038  0.121675  0.443863\n5  0.333674  1.494079 -0.205158\n6  0.313068 -0.854096 -2.552990"},"execution_count":200,"metadata":{},"output_type":"execute_result"}],"source":["df.fillna({1: 0.5, 2: 0})"]},{"cell_type":"code","execution_count":201,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.764052</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2.240893</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.950088</td>\n      <td>0.000000</td>\n      <td>-0.103219</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.410599</td>\n      <td>0.000000</td>\n      <td>1.454274</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.761038</td>\n      <td>0.121675</td>\n      <td>0.443863</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.333674</td>\n      <td>1.494079</td>\n      <td>-0.205158</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>0.313068</td>\n      <td>-0.854096</td>\n      <td>-2.552990</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          0         1         2\n0  1.764052  0.000000  0.000000\n1  2.240893  0.000000  0.000000\n2  0.950088  0.000000 -0.103219\n3  0.410599  0.000000  1.454274\n4  0.761038  0.121675  0.443863\n5  0.333674  1.494079 -0.205158\n6  0.313068 -0.854096 -2.552990"},"execution_count":201,"metadata":{},"output_type":"execute_result"}],"source":["df.fillna(0, inplace=True)\n","df"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["The same interpolation methods available for deciding which rows or columns to drop can be used for filling in data."]},{"cell_type":"code","execution_count":202,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.264556</td>\n      <td>0.774234</td>\n      <td>0.456150</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.568434</td>\n      <td>0.018790</td>\n      <td>0.617635</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.612096</td>\n      <td>NaN</td>\n      <td>0.943748</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.681820</td>\n      <td>NaN</td>\n      <td>0.437032</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.697631</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.670638</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          0         1         2\n0  0.264556  0.774234  0.456150\n1  0.568434  0.018790  0.617635\n2  0.612096       NaN  0.943748\n3  0.681820       NaN  0.437032\n4  0.697631       NaN       NaN\n5  0.670638       NaN       NaN"},"execution_count":202,"metadata":{},"output_type":"execute_result"}],"source":["df = pd.DataFrame(np.random.rand(6, 3))\n","df.iloc[2:, 1] = NA\n","df.iloc[4:, 2] = NA\n","df"]},{"cell_type":"code","execution_count":203,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.264556</td>\n      <td>0.774234</td>\n      <td>0.456150</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.568434</td>\n      <td>0.018790</td>\n      <td>0.617635</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.612096</td>\n      <td>0.018790</td>\n      <td>0.943748</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.681820</td>\n      <td>0.018790</td>\n      <td>0.437032</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.697631</td>\n      <td>0.018790</td>\n      <td>0.437032</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.670638</td>\n      <td>0.018790</td>\n      <td>0.437032</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          0         1         2\n0  0.264556  0.774234  0.456150\n1  0.568434  0.018790  0.617635\n2  0.612096  0.018790  0.943748\n3  0.681820  0.018790  0.437032\n4  0.697631  0.018790  0.437032\n5  0.670638  0.018790  0.437032"},"execution_count":203,"metadata":{},"output_type":"execute_result"}],"source":["df.fillna(method='ffill')  # 'ffill' = 'forward fill'"]},{"cell_type":"code","execution_count":204,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.264556</td>\n      <td>0.774234</td>\n      <td>0.456150</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.568434</td>\n      <td>0.018790</td>\n      <td>0.617635</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.612096</td>\n      <td>0.018790</td>\n      <td>0.943748</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.681820</td>\n      <td>0.018790</td>\n      <td>0.437032</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.697631</td>\n      <td>NaN</td>\n      <td>0.437032</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.670638</td>\n      <td>NaN</td>\n      <td>0.437032</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          0         1         2\n0  0.264556  0.774234  0.456150\n1  0.568434  0.018790  0.617635\n2  0.612096  0.018790  0.943748\n3  0.681820  0.018790  0.437032\n4  0.697631       NaN  0.437032\n5  0.670638       NaN  0.437032"},"execution_count":204,"metadata":{},"output_type":"execute_result"}],"source":["df.fillna(method='ffill', limit=2)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["## 7.2 Data transformation\n","\n","### Removing duplicates\n","\n","Duplicate rows can be found using the `duplicated()` method on a DataFrame."]},{"cell_type":"code","execution_count":205,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>two</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>two</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"    k1  k2\n0  one   1\n1  two   1\n2  one   2\n3  two   3\n4  one   3\n5  two   4\n6  two   4"},"execution_count":205,"metadata":{},"output_type":"execute_result"}],"source":["data = pd.DataFrame({\n","    'k1': ['one', 'two'] * 3 + ['two'],\n","    'k2': [1, 1, 2, 3, 3, 4, 4]\n","})\n","data"]},{"cell_type":"code","execution_count":206,"metadata":{},"outputs":[{"data":{"text/plain":"0    False\n1    False\n2    False\n3    False\n4    False\n5    False\n6     True\ndtype: bool"},"execution_count":206,"metadata":{},"output_type":"execute_result"}],"source":["data.duplicated()"]},{"cell_type":"code","execution_count":207,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>two</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"    k1  k2\n0  one   1\n1  two   1\n2  one   2\n3  two   3\n4  one   3\n5  two   4"},"execution_count":207,"metadata":{},"output_type":"execute_result"}],"source":["data.drop_duplicates()"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["It is also possible to specify the column by which to find duplicates."]},{"cell_type":"code","execution_count":208,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n      <th>v1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>two</td>\n      <td>4</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>two</td>\n      <td>4</td>\n      <td>6</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"    k1  k2  v1\n0  one   1   0\n1  two   1   1\n2  one   2   2\n3  two   3   3\n4  one   3   4\n5  two   4   5\n6  two   4   6"},"execution_count":208,"metadata":{},"output_type":"execute_result"}],"source":["data['v1'] = range(7)\n","data"]},{"cell_type":"code","execution_count":209,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n      <th>v1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"    k1  k2  v1\n0  one   1   0\n1  two   1   1"},"execution_count":209,"metadata":{},"output_type":"execute_result"}],"source":["data.drop_duplicates(['k1'])"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["By default, the first unique row is kept, though this can be changed to keeping the last one."]},{"cell_type":"code","execution_count":210,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n      <th>v1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>two</td>\n      <td>4</td>\n      <td>6</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"    k1  k2  v1\n0  one   1   0\n1  two   1   1\n2  one   2   2\n3  two   3   3\n4  one   3   4\n6  two   4   6"},"execution_count":210,"metadata":{},"output_type":"execute_result"}],"source":["data.drop_duplicates(['k1', 'k2'], keep='last')"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["### Transforming data using a function or mapping\n","\n","Often want to perform a transformation on an array, Series of column of a DataFrame."]},{"cell_type":"code","execution_count":211,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>food</th>\n      <th>ounces</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>bacon</td>\n      <td>4.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>pulled pork</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>bacon</td>\n      <td>12.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Pastrami</td>\n      <td>6.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>corned beef</td>\n      <td>7.5</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Bacon</td>\n      <td>8.0</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>pastrami</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>honey ham</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>nova lox</td>\n      <td>6.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          food  ounces\n0        bacon     4.0\n1  pulled pork     3.0\n2        bacon    12.0\n3     Pastrami     6.0\n4  corned beef     7.5\n5        Bacon     8.0\n6     pastrami     3.0\n7    honey ham     5.0\n8     nova lox     6.0"},"execution_count":211,"metadata":{},"output_type":"execute_result"}],"source":["data = pd.DataFrame({\n","    'food': ['bacon', 'pulled pork', 'bacon',\n","             'Pastrami', 'corned beef', 'Bacon',\n","             'pastrami', 'honey ham', 'nova lox'],\n","    'ounces': [4, 3, 12, 6, 7.5, 8, 3, 5, 6]\n","})\n","data"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["We can add a column to indicate the source animal for each food.\n","The `map` method on a Series accepts function or dictionary to apply to each element."]},{"cell_type":"code","execution_count":212,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>food</th>\n      <th>ounces</th>\n      <th>animal</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>bacon</td>\n      <td>4.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>pulled pork</td>\n      <td>3.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>bacon</td>\n      <td>12.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Pastrami</td>\n      <td>6.0</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>corned beef</td>\n      <td>7.5</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Bacon</td>\n      <td>8.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>pastrami</td>\n      <td>3.0</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>honey ham</td>\n      <td>5.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>nova lox</td>\n      <td>6.0</td>\n      <td>salmon</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          food  ounces  animal\n0        bacon     4.0     pig\n1  pulled pork     3.0     pig\n2        bacon    12.0     pig\n3     Pastrami     6.0     cow\n4  corned beef     7.5     cow\n5        Bacon     8.0     pig\n6     pastrami     3.0     cow\n7    honey ham     5.0     pig\n8     nova lox     6.0  salmon"},"execution_count":212,"metadata":{},"output_type":"execute_result"}],"source":["meat_to_animal = {\n","    'bacon': 'pig',\n","    'pulled pork': 'pig',\n","    'pastrami': 'cow',\n","    'corned beef': 'cow',\n","    'honey ham': 'pig',\n","    'nova lox': 'salmon'\n","}\n","\n","data['animal'] = data['food'].str.lower().map(meat_to_animal)\n","data"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["Alternatively, we could have passed a function."]},{"cell_type":"code","execution_count":213,"metadata":{},"outputs":[{"data":{"text/plain":"0       pig\n1       pig\n2       pig\n3       cow\n4       cow\n5       pig\n6       cow\n7       pig\n8    salmon\nName: food, dtype: object"},"execution_count":213,"metadata":{},"output_type":"execute_result"}],"source":["data['food'].map(lambda x: meat_to_animal[x.lower()])"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["### Replacing values\n","\n","The `replace(to_replace=x, value=y)` method replaces `x` with `y` in a Series."]},{"cell_type":"code","execution_count":214,"metadata":{},"outputs":[{"data":{"text/plain":"0       1.0\n1       NaN\n2       2.0\n3       NaN\n4   -1000.0\n5       3.0\ndtype: float64"},"execution_count":214,"metadata":{},"output_type":"execute_result"}],"source":["data = pd.Series([1.0, -999.0, 2.0, -999.0, -1000.0, 3.0])\n","data.replace(-999, np.nan)"]},{"cell_type":"code","execution_count":215,"metadata":{},"outputs":[{"data":{"text/plain":"0    1.0\n1    NaN\n2    2.0\n3    NaN\n4    NaN\n5    3.0\ndtype: float64"},"execution_count":215,"metadata":{},"output_type":"execute_result"}],"source":["data.replace([-999, -1000], np.nan)"]},{"cell_type":"code","execution_count":216,"metadata":{},"outputs":[{"data":{"text/plain":"0    1.0\n1    NaN\n2    2.0\n3    NaN\n4    0.0\n5    3.0\ndtype: float64"},"execution_count":216,"metadata":{},"output_type":"execute_result"}],"source":["data.replace([-999, -1000], [np.nan, 0])"]},{"cell_type":"code","execution_count":217,"metadata":{},"outputs":[{"data":{"text/plain":"0    1.0\n1    NaN\n2    2.0\n3    NaN\n4    0.0\n5    3.0\ndtype: float64"},"execution_count":217,"metadata":{},"output_type":"execute_result"}],"source":["data.replace({-999: np.nan, -1000: 0})"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["### Renaming axis indexes\n","\n","Axis labels can be transformed by a function or mapping, too.\n","This can be accomplished in-place."]},{"cell_type":"code","execution_count":218,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>three</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Ohio</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>Colorado</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>New York</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"          one  two  three  four\nOhio        0    1      2     3\nColorado    4    5      6     7\nNew York    8    9     10    11"},"execution_count":218,"metadata":{},"output_type":"execute_result"}],"source":["data = pd.DataFrame(np.arange(12).reshape((3, 4)),\n","                    index=['Ohio', 'Colorado', 'New York'],\n","                    columns=['one', 'two', 'three', 'four'])\n","data"]},{"cell_type":"code","execution_count":219,"metadata":{},"outputs":[{"data":{"text/plain":"Index(['OHIO', 'COLO', 'NEW '], dtype='object')"},"execution_count":219,"metadata":{},"output_type":"execute_result"}],"source":["transform = lambda x: x[:4].upper()\n","data.index.map(transform)"]},{"cell_type":"code","execution_count":220,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>three</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>OHIO</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>COLO</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>NEW</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"      one  two  three  four\nOHIO    0    1      2     3\nCOLO    4    5      6     7\nNEW     8    9     10    11"},"execution_count":220,"metadata":{},"output_type":"execute_result"}],"source":["data.index = data.index.map(transform)\n","data"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["The index and column names can be transformed by passing functions to the `rename()` method."]},{"cell_type":"code","execution_count":221,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ONE</th>\n      <th>TWO</th>\n      <th>THREE</th>\n      <th>FOUR</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Ohio</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>Colo</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>New</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"      ONE  TWO  THREE  FOUR\nOhio    0    1      2     3\nColo    4    5      6     7\nNew     8    9     10    11"},"execution_count":221,"metadata":{},"output_type":"execute_result"}],"source":["data.rename(index = str.title, columns = str.upper)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["One can rename a subset of the indices and column names using a dictionary mapping."]},{"cell_type":"code","execution_count":222,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>peekaboo</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>INDIANA</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>COLO</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>NEW</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"         one  two  peekaboo  four\nINDIANA    0    1         2     3\nCOLO       4    5         6     7\nNEW        8    9        10    11"},"execution_count":222,"metadata":{},"output_type":"execute_result"}],"source":["data.rename(index={'OHIO': 'INDIANA'},\n","            columns={'three': 'peekaboo'})"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["### Discretization and binning\n","\n","As an example, we want to bin the following age data into 18-25, 26-35, 36-60, 61 and older."]},{"cell_type":"code","execution_count":223,"metadata":{},"outputs":[{"data":{"text/plain":"[(18, 25], (18, 25], (18, 25], (25, 35], (18, 25], ..., (25, 35], (60, 100], (35, 60], (35, 60], (25, 35]]\nLength: 12\nCategories (4, interval[int64]): [(18, 25] < (25, 35] < (35, 60] < (60, 100]]"},"execution_count":223,"metadata":{},"output_type":"execute_result"}],"source":["ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]\n","bins = [18, 25, 35, 60, 100]\n","cats = pd.cut(ages, bins)\n","cats"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["The `cut()` function returns a `Categorical` object."]},{"cell_type":"code","execution_count":224,"metadata":{},"outputs":[{"data":{"text/plain":"array([0, 0, 0, 1, 0, 0, 2, 1, 3, 2, 2, 1], dtype=int8)"},"execution_count":224,"metadata":{},"output_type":"execute_result"}],"source":["cats.codes"]},{"cell_type":"code","execution_count":225,"metadata":{},"outputs":[{"data":{"text/plain":"IntervalIndex([(18, 25], (25, 35], (35, 60], (60, 100]],\n              closed='right',\n              dtype='interval[int64]')"},"execution_count":225,"metadata":{},"output_type":"execute_result"}],"source":["cats.categories"]},{"cell_type":"code","execution_count":226,"metadata":{},"outputs":[{"data":{"text/plain":"(18, 25]     5\n(35, 60]     3\n(25, 35]     3\n(60, 100]    1\ndtype: int64"},"execution_count":226,"metadata":{},"output_type":"execute_result"}],"source":["pd.value_counts(cats)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["We can dictate a few paramters for `cut()` such as `right`for setting the left of the boundary to be inclusive and the right to be exclusive (denoted in standard methematical notation)."]},{"cell_type":"code","execution_count":227,"metadata":{},"outputs":[{"data":{"text/plain":"[[18, 26), [18, 26), [18, 26), [26, 36), [18, 26), ..., [26, 36), [61, 100), [36, 61), [36, 61), [26, 36)]\nLength: 12\nCategories (4, interval[int64]): [[18, 26) < [26, 36) < [36, 61) < [61, 100)]"},"execution_count":227,"metadata":{},"output_type":"execute_result"}],"source":["pd.cut(ages, [18, 26, 36, 61, 100], right=False)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["The names of te bins can be specified."]},{"cell_type":"code","execution_count":228,"metadata":{},"outputs":[{"data":{"text/plain":"[Youth, Youth, Youth, YoungAdult, Youth, ..., YoungAdult, Senior, MiddleAged, MiddleAged, YoungAdult]\nLength: 12\nCategories (4, object): [Youth < YoungAdult < MiddleAged < Senior]"},"execution_count":228,"metadata":{},"output_type":"execute_result"}],"source":["group_names = ['Youth', 'YoungAdult', 'MiddleAged', 'Senior']\n","pd.cut(ages, bins, labels=group_names)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["Instead of specifying the bins, an integer can be passed for the number of equispaced bins to use."]},{"cell_type":"code","execution_count":229,"metadata":{},"outputs":[{"data":{"text/plain":"[(0.096, 0.32], (0.32, 0.54], (0.54, 0.77], (0.32, 0.54], (0.77, 0.99], ..., (0.096, 0.32], (0.096, 0.32], (0.32, 0.54], (0.77, 0.99], (0.096, 0.32]]\nLength: 20\nCategories (4, interval[float64]): [(0.096, 0.32] < (0.32, 0.54] < (0.54, 0.77] < (0.77, 0.99]]"},"execution_count":229,"metadata":{},"output_type":"execute_result"}],"source":["data = np.random.rand(20)\n","pd.cut(data, 4, precision=2)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["The `qcut()` function binds the data based on sample quantiles.\n","This results in roughly equal-sized bins."]},{"cell_type":"code","execution_count":230,"metadata":{},"outputs":[{"data":{"text/plain":"[(0.605, 2.759], (-0.0516, 0.605], (-0.0516, 0.605], (-0.0516, 0.605], (-0.0516, 0.605], ..., (-0.0516, 0.605], (-3.0469999999999997, -0.698], (-3.0469999999999997, -0.698], (-3.0469999999999997, -0.698], (-0.698, -0.0516]]\nLength: 1000\nCategories (4, interval[float64]): [(-3.0469999999999997, -0.698] < (-0.698, -0.0516] < (-0.0516, 0.605] < (0.605, 2.759]]"},"execution_count":230,"metadata":{},"output_type":"execute_result"}],"source":["data = np.random.randn(1000)\n","cats = pd.qcut(data, 4)  # cut into quartiles\n","cats"]},{"cell_type":"code","execution_count":231,"metadata":{},"outputs":[{"data":{"text/plain":"(0.605, 2.759]                   250\n(-0.0516, 0.605]                 250\n(-0.698, -0.0516]                250\n(-3.0469999999999997, -0.698]    250\ndtype: int64"},"execution_count":231,"metadata":{},"output_type":"execute_result"}],"source":["pd.value_counts(cats)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["The specific quantiles to use can also be passed to `qcut()`."]},{"cell_type":"code","execution_count":232,"metadata":{},"outputs":[{"data":{"text/plain":"[(-0.0516, 1.182], (-0.0516, 1.182], (-0.0516, 1.182], (-0.0516, 1.182], (-0.0516, 1.182], ..., (-0.0516, 1.182], (-1.28, -0.0516], (-1.28, -0.0516], (-1.28, -0.0516], (-1.28, -0.0516]]\nLength: 1000\nCategories (4, interval[float64]): [(-3.0469999999999997, -1.28] < (-1.28, -0.0516] < (-0.0516, 1.182] < (1.182, 2.759]]"},"execution_count":232,"metadata":{},"output_type":"execute_result"}],"source":["pd.qcut(data, [0, 0.1, 0.5, 0.9, 1.0])"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["### Detecting and filtering outliers\n","\n","Here is an example for detecting and filtering outliers in a data set."]},{"cell_type":"code","execution_count":233,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>0.014493</td>\n      <td>-0.005655</td>\n      <td>-0.002238</td>\n      <td>-0.041619</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>0.971579</td>\n      <td>0.999478</td>\n      <td>0.991132</td>\n      <td>0.990014</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>-3.007437</td>\n      <td>-3.116857</td>\n      <td>-3.392300</td>\n      <td>-3.740101</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>-0.654814</td>\n      <td>-0.671151</td>\n      <td>-0.637831</td>\n      <td>-0.763313</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>-0.010606</td>\n      <td>0.003705</td>\n      <td>-0.063378</td>\n      <td>0.011749</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>0.659244</td>\n      <td>0.693056</td>\n      <td>0.646954</td>\n      <td>0.637535</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>2.979976</td>\n      <td>3.801660</td>\n      <td>3.427539</td>\n      <td>2.929096</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"                 0            1            2            3\ncount  1000.000000  1000.000000  1000.000000  1000.000000\nmean      0.014493    -0.005655    -0.002238    -0.041619\nstd       0.971579     0.999478     0.991132     0.990014\nmin      -3.007437    -3.116857    -3.392300    -3.740101\n25%      -0.654814    -0.671151    -0.637831    -0.763313\n50%      -0.010606     0.003705    -0.063378     0.011749\n75%       0.659244     0.693056     0.646954     0.637535\nmax       2.979976     3.801660     3.427539     2.929096"},"execution_count":233,"metadata":{},"output_type":"execute_result"}],"source":["data = pd.DataFrame(np.random.randn(1000, 4))\n","data.describe()"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["Lets identify values greater than 3 in magnitude."]},{"cell_type":"code","execution_count":234,"metadata":{},"outputs":[{"data":{"text/plain":"598    3.427539\n656   -3.392300\nName: 2, dtype: float64"},"execution_count":234,"metadata":{},"output_type":"execute_result"}],"source":["col = data[2]\n","col[np.abs(col) > 3]"]},{"cell_type":"markdown","execution_count":103,"metadata":{},"outputs":[{"data":{"text/plain":"1000"},"execution_count":103,"metadata":{},"output_type":"execute_result"}],"source":["The rows with data greater than 3 in magnitide can be selected using the `any()` method."]},{"cell_type":"code","execution_count":235,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.049093</td>\n      <td>3.170975</td>\n      <td>0.189500</td>\n      <td>-1.348413</td>\n    </tr>\n    <tr>\n      <th>241</th>\n      <td>0.422819</td>\n      <td>-3.116857</td>\n      <td>0.644452</td>\n      <td>-1.913743</td>\n    </tr>\n    <tr>\n      <th>507</th>\n      <td>-0.347585</td>\n      <td>3.306574</td>\n      <td>-1.510200</td>\n      <td>0.203540</td>\n    </tr>\n    <tr>\n      <th>516</th>\n      <td>-0.126856</td>\n      <td>3.801660</td>\n      <td>2.315171</td>\n      <td>0.139827</td>\n    </tr>\n    <tr>\n      <th>598</th>\n      <td>-0.752582</td>\n      <td>0.045113</td>\n      <td>3.427539</td>\n      <td>0.604682</td>\n    </tr>\n    <tr>\n      <th>602</th>\n      <td>-0.553965</td>\n      <td>-3.006499</td>\n      <td>-0.047166</td>\n      <td>0.831878</td>\n    </tr>\n    <tr>\n      <th>656</th>\n      <td>-0.479297</td>\n      <td>-1.345508</td>\n      <td>-3.392300</td>\n      <td>0.155794</td>\n    </tr>\n    <tr>\n      <th>674</th>\n      <td>1.061435</td>\n      <td>-0.435034</td>\n      <td>0.657682</td>\n      <td>-3.740101</td>\n    </tr>\n    <tr>\n      <th>854</th>\n      <td>-3.007437</td>\n      <td>-2.330467</td>\n      <td>-0.567803</td>\n      <td>2.667322</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"            0         1         2         3\n0    1.049093  3.170975  0.189500 -1.348413\n241  0.422819 -3.116857  0.644452 -1.913743\n507 -0.347585  3.306574 -1.510200  0.203540\n516 -0.126856  3.801660  2.315171  0.139827\n598 -0.752582  0.045113  3.427539  0.604682\n602 -0.553965 -3.006499 -0.047166  0.831878\n656 -0.479297 -1.345508 -3.392300  0.155794\n674  1.061435 -0.435034  0.657682 -3.740101\n854 -3.007437 -2.330467 -0.567803  2.667322"},"execution_count":235,"metadata":{},"output_type":"execute_result"}],"source":["data[(np.abs(data) > 3).any(1)]"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["Values can be set using this boolean indexing."]},{"cell_type":"code","execution_count":236,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>0.014500</td>\n      <td>-0.006811</td>\n      <td>-0.002273</td>\n      <td>-0.040879</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>0.971556</td>\n      <td>0.994856</td>\n      <td>0.988474</td>\n      <td>0.987520</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>-3.000000</td>\n      <td>-3.000000</td>\n      <td>-3.000000</td>\n      <td>-3.000000</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>-0.654814</td>\n      <td>-0.671151</td>\n      <td>-0.637831</td>\n      <td>-0.763313</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>-0.010606</td>\n      <td>0.003705</td>\n      <td>-0.063378</td>\n      <td>0.011749</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>0.659244</td>\n      <td>0.693056</td>\n      <td>0.646954</td>\n      <td>0.637535</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>2.979976</td>\n      <td>3.000000</td>\n      <td>3.000000</td>\n      <td>2.929096</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"                 0            1            2            3\ncount  1000.000000  1000.000000  1000.000000  1000.000000\nmean      0.014500    -0.006811    -0.002273    -0.040879\nstd       0.971556     0.994856     0.988474     0.987520\nmin      -3.000000    -3.000000    -3.000000    -3.000000\n25%      -0.654814    -0.671151    -0.637831    -0.763313\n50%      -0.010606     0.003705    -0.063378     0.011749\n75%       0.659244     0.693056     0.646954     0.637535\nmax       2.979976     3.000000     3.000000     2.929096"},"execution_count":236,"metadata":{},"output_type":"execute_result"}],"source":["data[np.abs(data) > 3] = np.sign(data) * 3\n","data.describe()"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["### Permutation and random sampling\n","\n","Permuting a Series or rows of a DataFrame is easily accomplished using `numpy.random.permutation()`.\n","Providing this function with the length of the axis to permute returns a new indexing to use for reordering."]},{"cell_type":"code","execution_count":237,"metadata":{},"outputs":[{"data":{"text/plain":"array([0, 3, 2, 4, 1])"},"execution_count":237,"metadata":{},"output_type":"execute_result"}],"source":["df = pd.DataFrame(np.arange(5 * 4).reshape((5, 4)))\n","sampler = np.random.permutation(5)\n","sampler"]},{"cell_type":"code","execution_count":238,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>12</td>\n      <td>13</td>\n      <td>14</td>\n      <td>15</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>16</td>\n      <td>17</td>\n      <td>18</td>\n      <td>19</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"    0   1   2   3\n0   0   1   2   3\n3  12  13  14  15\n2   8   9  10  11\n4  16  17  18  19\n1   4   5   6   7"},"execution_count":238,"metadata":{},"output_type":"execute_result"}],"source":["df.take(sampler)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["A sample of a DataFrame can be taken with or without replacement."]},{"cell_type":"code","execution_count":239,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"   0  1   2   3\n2  8  9  10  11\n0  0  1   2   3\n1  4  5   6   7"},"execution_count":239,"metadata":{},"output_type":"execute_result"}],"source":["df.sample(n=3)"]},{"cell_type":"code","execution_count":240,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"   0  1  2  3\n1  4  5  6  7\n1  4  5  6  7\n0  0  1  2  3\n1  4  5  6  7\n0  0  1  2  3\n0  0  1  2  3"},"execution_count":240,"metadata":{},"output_type":"execute_result"}],"source":["df.sample(n=6, replace=True)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["### Computing indicator or dummy variables\n","\n","This is often a useful tool for statistical analysis or machine learning."]},{"cell_type":"code","execution_count":241,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>key</th>\n      <th>data1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>b</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>b</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>a</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>c</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>a</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>b</td>\n      <td>5</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"  key  data1\n0   b      0\n1   b      1\n2   a      2\n3   c      3\n4   a      4\n5   b      5"},"execution_count":241,"metadata":{},"output_type":"execute_result"}],"source":["df = pd.DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'b'],\n","                   'data1': range(6)})\n","df"]},{"cell_type":"code","execution_count":242,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>a</th>\n      <th>b</th>\n      <th>c</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"   a  b  c\n0  0  1  0\n1  0  1  0\n2  1  0  0\n3  0  0  1\n4  1  0  0\n5  0  1  0"},"execution_count":242,"metadata":{},"output_type":"execute_result"}],"source":["pd.get_dummies(df['key'])"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["Somtimes if it useful to include a prefix in the column names."]},{"cell_type":"code","execution_count":243,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>data1</th>\n      <th>key_a</th>\n      <th>key_b</th>\n      <th>key_c</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>5</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>","text/plain":"   data1  key_a  key_b  key_c\n0      0      0      1      0\n1      1      0      1      0\n2      2      1      0      0\n3      3      0      0      1\n4      4      1      0      0\n5      5      0      1      0"},"execution_count":243,"metadata":{},"output_type":"execute_result"}],"source":["dummies = pd.get_dummies(df['key'], prefix='key')\n","df_with_dummy = df[['data1']].join(dummies)\n","df_with_dummy"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["Creating a dummy matrix is more complicated if a row of a DataFrame belongs to multiple categories.\n","For an example, we used the MovieLens 1M data set."]},{"cell_type":"code","execution_count":244,"metadata":{},"outputs":[{"name":"stderr","output_type":"stream","text":"/opt/anaconda3/envs/daysOfCode-env/lib/python3.7/site-packages/ipykernel_launcher.py:3: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n  This is separate from the ipykernel package so we can avoid doing imports until\n"},{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>movie_id</th>\n      <th>title</th>\n      <th>genres</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>Toy Story (1995)</td>\n      <td>Animation|Children's|Comedy</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>Jumanji (1995)</td>\n      <td>Adventure|Children's|Fantasy</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>Grumpier Old Men (1995)</td>\n      <td>Comedy|Romance</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>4</td>\n      <td>Waiting to Exhale (1995)</td>\n      <td>Comedy|Drama</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>5</td>\n      <td>Father of the Bride Part II (1995)</td>\n      <td>Comedy</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>3878</th>\n      <td>3948</td>\n      <td>Meet the Parents (2000)</td>\n      <td>Comedy</td>\n    </tr>\n    <tr>\n      <th>3879</th>\n      <td>3949</td>\n      <td>Requiem for a Dream (2000)</td>\n      <td>Drama</td>\n    </tr>\n    <tr>\n      <th>3880</th>\n      <td>3950</td>\n      <td>Tigerland (2000)</td>\n      <td>Drama</td>\n    </tr>\n    <tr>\n      <th>3881</th>\n      <td>3951</td>\n      <td>Two Family House (2000)</td>\n      <td>Drama</td>\n    </tr>\n    <tr>\n      <th>3882</th>\n      <td>3952</td>\n      <td>Contender, The (2000)</td>\n      <td>Drama|Thriller</td>\n    </tr>\n  </tbody>\n</table>\n<p>3883 rows × 3 columns</p>\n</div>","text/plain":"      movie_id                               title  \\\n0            1                    Toy Story (1995)   \n1            2                      Jumanji (1995)   \n2            3             Grumpier Old Men (1995)   \n3            4            Waiting to Exhale (1995)   \n4            5  Father of the Bride Part II (1995)   \n...        ...                                 ...   \n3878      3948             Meet the Parents (2000)   \n3879      3949          Requiem for a Dream (2000)   \n3880      3950                    Tigerland (2000)   \n3881      3951             Two Family House (2000)   \n3882      3952               Contender, The (2000)   \n\n                            genres  \n0      Animation|Children's|Comedy  \n1     Adventure|Children's|Fantasy  \n2                   Comedy|Romance  \n3                     Comedy|Drama  \n4                           Comedy  \n...                            ...  \n3878                        Comedy  \n3879                         Drama  \n3880                         Drama  \n3881                         Drama  \n3882                Drama|Thriller  \n\n[3883 rows x 3 columns]"},"execution_count":244,"metadata":{},"output_type":"execute_result"}],"source":["mnames = ['movie_id', 'title', 'genres']\n","movies = pd.read_table('assets/datasets/movielens/movies.dat', sep='::',\n","                       header=None, names = mnames)\n","movies"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["The goal of the next set of operations is to add an indicator variable for genre."]},{"cell_type":"code","execution_count":245,"metadata":{},"outputs":[{"data":{"text/plain":"array(['Animation', \"Children's\", 'Comedy', 'Adventure', 'Fantasy',\n       'Romance', 'Drama', 'Action', 'Crime', 'Thriller', 'Horror',\n       'Sci-Fi', 'Documentary', 'War', 'Musical', 'Mystery', 'Film-Noir',\n       'Western'], dtype=object)"},"execution_count":245,"metadata":{},"output_type":"execute_result"}],"source":["all_genres = []\n","for x in movies.genres:\n","    all_genres.extend(x.split('|'))\n","\n","genres = pd.unique(all_genres)\n","genres"]},{"cell_type":"code","execution_count":246,"metadata":{},"outputs":[{"data":{"text/plain":"['Animation', \"Children's\", 'Comedy']"},"execution_count":246,"metadata":{},"output_type":"execute_result"}],"source":["zero_matrix = np.zeros((len(movies), len(genres)))\n","dummies = pd.DataFrame(zero_matrix, columns=genres)\n","gen = movies.genres[0]\n","gen.split('|')"]},{"cell_type":"code","execution_count":247,"metadata":{},"outputs":[{"data":{"text/plain":"array([0, 1, 2])"},"execution_count":247,"metadata":{},"output_type":"execute_result"}],"source":["dummies.columns.get_indexer(gen.split('|'))"]},{"cell_type":"code","execution_count":248,"metadata":{},"outputs":[{"data":{"text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Animation</th>\n      <th>Children's</th>\n      <th>Comedy</th>\n      <th>Adventure</th>\n      <th>Fantasy</th>\n      <th>Romance</th>\n      <th>Drama</th>\n      <th>Action</th>\n      <th>Crime</th>\n      <th>Thriller</th>\n      <th>Horror</th>\n      <th>Sci-Fi</th>\n      <th>Documentary</th>\n      <th>War</th>\n      <th>Musical</th>\n      <th>Mystery</th>\n      <th>Film-Noir</th>\n      <th>Western</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>3878</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3879</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3880</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3881</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3882</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>3883 rows × 18 columns</p>\n</div>","text/plain":"      Animation  Children's  Comedy  Adventure  Fantasy  Romance  Drama  \\\n0           1.0         1.0     1.0        0.0      0.0      0.0    0.0   \n1           0.0         1.0     0.0        1.0      1.0      0.0    0.0   \n2           0.0         0.0     1.0        0.0      0.0      1.0    0.0   \n3           0.0         0.0     1.0        0.0      0.0      0.0    1.0   \n4           0.0         0.0     1.0        0.0      0.0      0.0    0.0   \n...         ...         ...     ...        ...      ...      ...    ...   \n3878        0.0         0.0     1.0        0.0      0.0      0.0    0.0   \n3879        0.0         0.0     0.0        0.0      0.0      0.0    1.0   \n3880        0.0         0.0     0.0        0.0      0.0      0.0    1.0   \n3881        0.0         0.0     0.0        0.0      0.0      0.0    1.0   \n3882        0.0         0.0     0.0        0.0      0.0      0.0    1.0   \n\n      Action  Crime  Thriller  Horror  Sci-Fi  Documentary  War  Musical  \\\n0        0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n1        0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n2        0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n3        0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n4        0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n...      ...    ...       ...     ...     ...          ...  ...      ...   \n3878     0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n3879     0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n3880     0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n3881     0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n3882     0.0    0.0       1.0     0.0     0.0          0.0  0.0      0.0   \n\n      Mystery  Film-Noir  Western  \n0         0.0        0.0      0.0  \n1         0.0        0.0      0.0  \n2         0.0        0.0      0.0  \n3         0.0        0.0      0.0  \n4         0.0        0.0      0.0  \n...       ...        ...      ...  \n3878      0.0        0.0      0.0  \n3879      0.0        0.0      0.0  \n3880      0.0        0.0      0.0  \n3881      0.0        0.0      0.0  \n3882      0.0        0.0      0.0  \n\n[3883 rows x 18 columns]"},"execution_count":248,"metadata":{},"output_type":"execute_result"}],"source":["for i, gen in enumerate(movies.genres):\n","    indices = dummies.columns.get_indexer(gen.split('|'))\n","    dummies.iloc[i, indices] = 1\n","\n","dummies"]},{"cell_type":"code","execution_count":249,"metadata":{},"outputs":[{"data":{"text/plain":"movie_id                                       1\ntitle                           Toy Story (1995)\ngenres               Animation|Children's|Comedy\nGenre_Animation                                1\nGenre_Children's                               1\nGenre_Comedy                                   1\nGenre_Adventure                                0\nGenre_Fantasy                                  0\nGenre_Romance                                  0\nGenre_Drama                                    0\nGenre_Action                                   0\nGenre_Crime                                    0\nGenre_Thriller                                 0\nGenre_Horror                                   0\nGenre_Sci-Fi                                   0\nGenre_Documentary                              0\nGenre_War                                      0\nGenre_Musical                                  0\nGenre_Mystery                                  0\nGenre_Film-Noir                                0\nGenre_Western                                  0\nName: 0, dtype: object"},"execution_count":249,"metadata":{},"output_type":"execute_result"}],"source":["movies_windic = movies.join(dummies.add_prefix('Genre_'))\n","movies_windic.iloc[0]"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["## 7.3 String manipulation\n","\n","pandas builds on top of Python's rich assortment of string manipulations and object methods by applying them to arrays of strings.\n","\n","### String object methods\n","\n","Here are a few commonly used string methods built into Python."]},{"cell_type":"code","execution_count":250,"metadata":{},"outputs":[{"data":{"text/plain":"['a', 'b', '  guido']"},"execution_count":250,"metadata":{},"output_type":"execute_result"}],"source":["val = 'a,b,  guido'\n","val.split(',')"]},{"cell_type":"code","execution_count":251,"metadata":{},"outputs":[{"data":{"text/plain":"['a', 'b', 'guido']"},"execution_count":251,"metadata":{},"output_type":"execute_result"}],"source":["pieces = [x.strip() for x in val.split(',')]\n","pieces"]},{"cell_type":"code","execution_count":252,"metadata":{},"outputs":[{"data":{"text/plain":"'a::b::guido'"},"execution_count":252,"metadata":{},"output_type":"execute_result"}],"source":["first, second, third = pieces\n","first + '::' + second + '::' + third"]},{"cell_type":"code","execution_count":253,"metadata":{},"outputs":[{"data":{"text/plain":"'a::b::guido'"},"execution_count":253,"metadata":{},"output_type":"execute_result"}],"source":["'::'.join(pieces)"]},{"cell_type":"code","execution_count":254,"metadata":{},"outputs":[{"data":{"text/plain":"True"},"execution_count":254,"metadata":{},"output_type":"execute_result"}],"source":["'guido' in val"]},{"cell_type":"code","execution_count":255,"metadata":{},"outputs":[{"data":{"text/plain":"1"},"execution_count":255,"metadata":{},"output_type":"execute_result"}],"source":["val.index(',')"]},{"cell_type":"code","execution_count":256,"metadata":{},"outputs":[{"data":{"text/plain":"-1"},"execution_count":256,"metadata":{},"output_type":"execute_result"}],"source":["val.find(':')"]},{"cell_type":"code","execution_count":257,"metadata":{},"outputs":[{"data":{"text/plain":"3"},"execution_count":257,"metadata":{},"output_type":"execute_result"}],"source":["val.rfind(',')"]},{"cell_type":"code","execution_count":258,"metadata":{},"outputs":[{"data":{"text/plain":"2"},"execution_count":258,"metadata":{},"output_type":"execute_result"}],"source":["val.count(',')"]},{"cell_type":"code","execution_count":259,"metadata":{},"outputs":[{"data":{"text/plain":"'a::b::  guido'"},"execution_count":259,"metadata":{},"output_type":"execute_result"}],"source":["val.replace(',', '::')"]},{"cell_type":"code","execution_count":260,"metadata":{},"outputs":[{"data":{"text/plain":"'ab  guido'"},"execution_count":260,"metadata":{},"output_type":"execute_result"}],"source":["val.replace(',', '')"]},{"cell_type":"code","execution_count":261,"metadata":{},"outputs":[{"data":{"text/plain":"'A,B,  GUIDO'"},"execution_count":261,"metadata":{},"output_type":"execute_result"}],"source":["upper_val = val.upper()\n","upper_val"]},{"cell_type":"code","execution_count":262,"metadata":{},"outputs":[{"data":{"text/plain":"'a,b,  guido'"},"execution_count":262,"metadata":{},"output_type":"execute_result"}],"source":["upper_val.lower()"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["### Regular expressions\n","\n","A *regex* provides a powerful way to identify patterns in strings.\n","Python's built in *re* module is responsible for applying regular expressions to strings.\n","The functions in *re* are in one of three categories: pattern matching, substitution, and splitting.\n","\n","Below are a bunch of expample os using regular expressions for various tasks.\n","The first is to split a string with a variable number of white spaces."]},{"cell_type":"code","execution_count":263,"metadata":{},"outputs":[{"data":{"text/plain":"['foo', 'bat', 'baz', 'qux']"},"execution_count":263,"metadata":{},"output_type":"execute_result"}],"source":["import re\n","\n","text = \"foo   bat\\t baz \\tqux\"\n","re.split('\\s+', text)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["When `re.split('\\s+', text)` was called, the regular expression was first compiled.\n","This can be done manually, creating a new object that can be used for further matching."]},{"cell_type":"code","execution_count":264,"metadata":{},"outputs":[{"data":{"text/plain":"['foo', 'bat', 'baz', 'qux']"},"execution_count":264,"metadata":{},"output_type":"execute_result"}],"source":["regex = re.compile('\\s+')\n","regex.split(text)"]},{"cell_type":"code","execution_count":265,"metadata":{},"outputs":[{"data":{"text/plain":"['   ', '\\t ', ' \\t']"},"execution_count":265,"metadata":{},"output_type":"execute_result"}],"source":["regex.findall(text)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["Here is an example with a larger string and pattern.\n","The `re.IGNORECASE` makes the regex case-insensitive."]},{"cell_type":"code","execution_count":266,"metadata":{},"outputs":[{"data":{"text/plain":"['dave@google.com', 'steve@gmail.com', 'rob@gmail.com', 'ryan@yahoo.com']"},"execution_count":266,"metadata":{},"output_type":"execute_result"}],"source":["text = \"\"\"Dave dave@google.com\n","Steve steve@gmail.com\n","Rob rob@gmail.com\n","Ryan ryan@yahoo.com\n","\"\"\"\n","pattern = r'[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}'\n","\n","regex = re.compile(pattern, flags=re.IGNORECASE)\n","\n","regex.findall(text)"]},{"cell_type":"code","execution_count":267,"metadata":{},"outputs":[{"data":{"text/plain":"<re.Match object; span=(5, 20), match='dave@google.com'>"},"execution_count":267,"metadata":{},"output_type":"execute_result"}],"source":["m = regex.search(text)\n","m"]},{"cell_type":"code","execution_count":268,"metadata":{},"outputs":[{"data":{"text/plain":"'dave@google.com'"},"execution_count":268,"metadata":{},"output_type":"execute_result"}],"source":["text[m.start():m.end()]"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["`match()` only checks for the pattern at the begininng of the string."]},{"cell_type":"code","execution_count":269,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":"None\n"}],"source":["print(regex.match(text))"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["The `sub()` function substitutes another string at the points of pattern match."]},{"cell_type":"code","execution_count":270,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":"Dave REDACTED\nSteve REDACTED\nRob REDACTED\nRyan REDACTED\n\n"}],"source":["print(regex.sub('REDACTED', text))"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["It is possible to find and segment a regular expression into groups using parantheses."]},{"cell_type":"code","execution_count":271,"metadata":{},"outputs":[{"data":{"text/plain":"('wesm', 'bright', 'net')"},"execution_count":271,"metadata":{},"output_type":"execute_result"}],"source":["pattern = r'([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\.([A-Z]{2,4})'\n","regex = re.compile(pattern, flags=re.IGNORECASE)\n","m = regex.match('wesm@bright.net')\n","m.groups()"]},{"cell_type":"code","execution_count":272,"metadata":{},"outputs":[{"data":{"text/plain":"[('dave', 'google', 'com'),\n ('steve', 'gmail', 'com'),\n ('rob', 'gmail', 'com'),\n ('ryan', 'yahoo', 'com')]"},"execution_count":272,"metadata":{},"output_type":"execute_result"}],"source":["regex.findall(text)"]},{"cell_type":"code","execution_count":273,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":"Dave Username: dave, Domain: google, Suffix: com\nSteve Username: steve, Domain: gmail, Suffix: com\nRob Username: rob, Domain: gmail, Suffix: com\nRyan Username: ryan, Domain: yahoo, Suffix: com\n\n"}],"source":["print(regex.sub(r'Username: \\1, Domain: \\2, Suffix: \\3', text))"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["### Vectorized string functions in pandas"]},{"cell_type":"code","execution_count":274,"metadata":{},"outputs":[{"data":{"text/plain":"Dave     dave@google.com\nSteve    steve@gmail.com\nRob        rob@gmail.com\nWes                  NaN\ndtype: object"},"execution_count":274,"metadata":{},"output_type":"execute_result"}],"source":["data = {\n","    'Dave': 'dave@google.com', 'Steve': 'steve@gmail.com',\n","    'Rob': 'rob@gmail.com', 'Wes': np.nan\n","}\n","data = pd.Series(data)\n","data"]},{"cell_type":"code","execution_count":275,"metadata":{},"outputs":[{"data":{"text/plain":"Dave     False\nSteve    False\nRob      False\nWes       True\ndtype: bool"},"execution_count":275,"metadata":{},"output_type":"execute_result"}],"source":["data.isnull()"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["Series has an `str` attribute for applying regular expressions over all values, skipping missing values (`NaN`)."]},{"cell_type":"code","execution_count":276,"metadata":{},"outputs":[{"data":{"text/plain":"Dave     False\nSteve     True\nRob       True\nWes        NaN\ndtype: object"},"execution_count":276,"metadata":{},"output_type":"execute_result"}],"source":["data.str.contains('gmail')"]},{"cell_type":"code","execution_count":277,"metadata":{},"outputs":[{"data":{"text/plain":"Dave     [(dave, google, com)]\nSteve    [(steve, gmail, com)]\nRob        [(rob, gmail, com)]\nWes                        NaN\ndtype: object"},"execution_count":277,"metadata":{},"output_type":"execute_result"}],"source":["pattern = r'([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\.([A-Z]{2,4})'\n","data.str.findall(pattern, flags=re.IGNORECASE)"]},{"cell_type":"markdown","execution_count":null,"metadata":{},"outputs":[],"source":["There are mutliple ways to do vectorized element retrieval."]},{"cell_type":"code","execution_count":278,"metadata":{},"outputs":[{"data":{"text/plain":"Dave     True\nSteve    True\nRob      True\nWes       NaN\ndtype: object"},"execution_count":278,"metadata":{},"output_type":"execute_result"}],"source":["matches = data.str.match(pattern, flags=re.IGNORECASE)\n","matches"]},{"cell_type":"code","execution_count":279,"metadata":{},"outputs":[{"data":{"text/plain":"Dave     dave@\nSteve    steve\nRob      rob@g\nWes        NaN\ndtype: object"},"execution_count":279,"metadata":{},"output_type":"execute_result"}],"source":["data.str[:5]"]}],"nbformat":4,"nbformat_minor":2,"metadata":{"language_info":{"name":"python","codemirror_mode":{"name":"ipython","version":3}},"orig_nbformat":2,"file_extension":".py","mimetype":"text/x-python","name":"python","npconvert_exporter":"python","pygments_lexer":"ipython3","version":3}}