dataset-3.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "language_info": {
   "name": "python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   }
  },
  "orig_nbformat": 2,
  "file_extension": ".py",
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "labor_market_statistics = pd.read_csv('Datasets/labor_market_statistics.csv')\n",
    "labor_market_statistics_legend = pd.read_csv('Datasets/Code Legends/labor_market_stats_legend.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Year</th>\n      <th>Month</th>\n      <th>YBZY</th>\n      <th>YBZR</th>\n      <th>YBZQ</th>\n      <th>YBZT</th>\n      <th>YBZS</th>\n      <th>YBZV</th>\n      <th>YBZU</th>\n      <th>YBZW</th>\n      <th>...</th>\n      <th>LF3M</th>\n      <th>AIYW</th>\n      <th>LF3T</th>\n      <th>LF45</th>\n      <th>YCGN</th>\n      <th>AIYX</th>\n      <th>DZ36</th>\n      <th>LF3S</th>\n      <th>LF44</th>\n      <th>YCGM</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>0</td>\n      <td>1992</td>\n      <td>YEAR</td>\n      <td>38376.0</td>\n      <td>88044.0</td>\n      <td>24636.0</td>\n      <td>37656.0</td>\n      <td>50388.0</td>\n      <td>64752.0</td>\n      <td>118176.0</td>\n      <td>53424.0</td>\n      <td>...</td>\n      <td>71.5</td>\n      <td>6.0</td>\n      <td>67.7</td>\n      <td>60.8</td>\n      <td>6072.0</td>\n      <td>6.6</td>\n      <td>NaN</td>\n      <td>71.3</td>\n      <td>57.9</td>\n      <td>9264.0</td>\n    </tr>\n    <tr>\n      <td>1</td>\n      <td>1993</td>\n      <td>YEAR</td>\n      <td>38184.0</td>\n      <td>89076.0</td>\n      <td>23628.0</td>\n      <td>38556.0</td>\n      <td>50520.0</td>\n      <td>64968.0</td>\n      <td>119064.0</td>\n      <td>54096.0</td>\n      <td>...</td>\n      <td>70.8</td>\n      <td>5.2</td>\n      <td>67.5</td>\n      <td>60.8</td>\n      <td>6132.0</td>\n      <td>6.2</td>\n      <td>NaN</td>\n      <td>71.3</td>\n      <td>58.4</td>\n      <td>9360.0</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>1994</td>\n      <td>YEAR</td>\n      <td>38400.0</td>\n      <td>89784.0</td>\n      <td>22248.0</td>\n      <td>39024.0</td>\n      <td>50772.0</td>\n      <td>65016.0</td>\n      <td>119376.0</td>\n      <td>54360.0</td>\n      <td>...</td>\n      <td>71.5</td>\n      <td>5.2</td>\n      <td>69.3</td>\n      <td>61.2</td>\n      <td>5568.0</td>\n      <td>6.1</td>\n      <td>NaN</td>\n      <td>70.5</td>\n      <td>57.0</td>\n      <td>8592.0</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>1995</td>\n      <td>YEAR</td>\n      <td>38436.0</td>\n      <td>90072.0</td>\n      <td>21516.0</td>\n      <td>39492.0</td>\n      <td>50568.0</td>\n      <td>65604.0</td>\n      <td>120468.0</td>\n      <td>54864.0</td>\n      <td>...</td>\n      <td>71.4</td>\n      <td>5.2</td>\n      <td>69.8</td>\n      <td>60.3</td>\n      <td>5016.0</td>\n      <td>6.1</td>\n      <td>NaN</td>\n      <td>72.2</td>\n      <td>58.4</td>\n      <td>7908.0</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>1996</td>\n      <td>YEAR</td>\n      <td>39192.0</td>\n      <td>89976.0</td>\n      <td>20952.0</td>\n      <td>39780.0</td>\n      <td>50184.0</td>\n      <td>65772.0</td>\n      <td>121524.0</td>\n      <td>55752.0</td>\n      <td>...</td>\n      <td>72.1</td>\n      <td>5.7</td>\n      <td>70.0</td>\n      <td>61.6</td>\n      <td>4656.0</td>\n      <td>6.8</td>\n      <td>NaN</td>\n      <td>72.9</td>\n      <td>58.5</td>\n      <td>7392.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 255 columns</p>\n</div>",
      "text/plain": "   Year Month     YBZY     YBZR     YBZQ     YBZT     YBZS     YBZV      YBZU  \\\n0  1992  YEAR  38376.0  88044.0  24636.0  37656.0  50388.0  64752.0  118176.0   \n1  1993  YEAR  38184.0  89076.0  23628.0  38556.0  50520.0  64968.0  119064.0   \n2  1994  YEAR  38400.0  89784.0  22248.0  39024.0  50772.0  65016.0  119376.0   \n3  1995  YEAR  38436.0  90072.0  21516.0  39492.0  50568.0  65604.0  120468.0   \n4  1996  YEAR  39192.0  89976.0  20952.0  39780.0  50184.0  65772.0  121524.0   \n\n      YBZW  ...  LF3M  AIYW  LF3T  LF45    YCGN  AIYX  DZ36  LF3S  LF44  \\\n0  53424.0  ...  71.5   6.0  67.7  60.8  6072.0   6.6   NaN  71.3  57.9   \n1  54096.0  ...  70.8   5.2  67.5  60.8  6132.0   6.2   NaN  71.3  58.4   \n2  54360.0  ...  71.5   5.2  69.3  61.2  5568.0   6.1   NaN  70.5  57.0   \n3  54864.0  ...  71.4   5.2  69.8  60.3  5016.0   6.1   NaN  72.2  58.4   \n4  55752.0  ...  72.1   5.7  70.0  61.6  4656.0   6.8   NaN  72.9  58.5   \n\n     YCGM  \n0  9264.0  \n1  9360.0  \n2  8592.0  \n3  7908.0  \n4  7392.0  \n\n[5 rows x 255 columns]"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labor_market_statistics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Title</th>\n      <th>CDID</th>\n      <th>Unit</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>0</td>\n      <td>LFS: Economically active: UK: Male: Aged 50-64...</td>\n      <td>YBZY</td>\n      <td>000's</td>\n    </tr>\n    <tr>\n      <td>1</td>\n      <td>LFS: Economically active: UK: All: Aged 25-34:...</td>\n      <td>YBZR</td>\n      <td>000's</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>LFS: Economically active: UK: Female: Aged 18-...</td>\n      <td>YBZQ</td>\n      <td>000's</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>LFS: Economically active: UK: Female: Aged 25-...</td>\n      <td>YBZT</td>\n      <td>000's</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>LFS: Economically active: UK: Male: Aged 25-34...</td>\n      <td>YBZS</td>\n      <td>000's</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "                                               Title  CDID   Unit\n0  LFS: Economically active: UK: Male: Aged 50-64...  YBZY  000's\n1  LFS: Economically active: UK: All: Aged 25-34:...  YBZR  000's\n2  LFS: Economically active: UK: Female: Aged 18-...  YBZQ  000's\n3  LFS: Economically active: UK: Female: Aged 25-...  YBZT  000's\n4  LFS: Economically active: UK: Male: Aged 25-34...  YBZS  000's"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labor_market_statistics_legend.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Year</th>\n      <th>YBZY</th>\n      <th>YBZR</th>\n      <th>YBZQ</th>\n      <th>YBZT</th>\n      <th>YBZS</th>\n      <th>YBZV</th>\n      <th>YBZU</th>\n      <th>YBZW</th>\n      <th>JPC3</th>\n      <th>...</th>\n      <th>LF3M</th>\n      <th>AIYW</th>\n      <th>LF3T</th>\n      <th>LF45</th>\n      <th>YCGN</th>\n      <th>AIYX</th>\n      <th>DZ36</th>\n      <th>LF3S</th>\n      <th>LF44</th>\n      <th>YCGM</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>count</td>\n      <td>324.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>218.000000</td>\n      <td>...</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>74.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n      <td>322.000000</td>\n    </tr>\n    <tr>\n      <td>mean</td>\n      <td>2009.388889</td>\n      <td>10769.878882</td>\n      <td>18642.186335</td>\n      <td>4906.537267</td>\n      <td>8477.003106</td>\n      <td>10164.990683</td>\n      <td>15404.714286</td>\n      <td>28979.785714</td>\n      <td>13575.086957</td>\n      <td>2.550917</td>\n      <td>...</td>\n      <td>74.414907</td>\n      <td>9.008696</td>\n      <td>71.050000</td>\n      <td>66.234161</td>\n      <td>613.080745</td>\n      <td>10.232609</td>\n      <td>131.891892</td>\n      <td>73.667081</td>\n      <td>63.912422</td>\n      <td>1079.307453</td>\n    </tr>\n    <tr>\n      <td>std</td>\n      <td>5.767691</td>\n      <td>12388.813285</td>\n      <td>22382.091688</td>\n      <td>5795.811994</td>\n      <td>10105.938143</td>\n      <td>12281.160677</td>\n      <td>18046.810211</td>\n      <td>33833.938916</td>\n      <td>15791.136565</td>\n      <td>0.686824</td>\n      <td>...</td>\n      <td>1.887397</td>\n      <td>1.590916</td>\n      <td>1.731462</td>\n      <td>2.220511</td>\n      <td>915.699909</td>\n      <td>1.724866</td>\n      <td>7.819413</td>\n      <td>1.461757</td>\n      <td>2.689109</td>\n      <td>1520.665722</td>\n    </tr>\n    <tr>\n      <td>min</td>\n      <td>1992.000000</td>\n      <td>3661.000000</td>\n      <td>6604.000000</td>\n      <td>1708.000000</td>\n      <td>3001.000000</td>\n      <td>3597.000000</td>\n      <td>5659.000000</td>\n      <td>10605.000000</td>\n      <td>4925.000000</td>\n      <td>1.300000</td>\n      <td>...</td>\n      <td>70.300000</td>\n      <td>5.200000</td>\n      <td>67.500000</td>\n      <td>60.300000</td>\n      <td>132.000000</td>\n      <td>6.100000</td>\n      <td>116.800000</td>\n      <td>70.500000</td>\n      <td>57.000000</td>\n      <td>248.000000</td>\n    </tr>\n    <tr>\n      <td>25%</td>\n      <td>2005.000000</td>\n      <td>4086.250000</td>\n      <td>6856.250000</td>\n      <td>1876.250000</td>\n      <td>3117.500000</td>\n      <td>3747.250000</td>\n      <td>5896.000000</td>\n      <td>11171.250000</td>\n      <td>5284.500000</td>\n      <td>2.100000</td>\n      <td>...</td>\n      <td>73.525000</td>\n      <td>8.100000</td>\n      <td>69.725000</td>\n      <td>64.800000</td>\n      <td>181.250000</td>\n      <td>9.225000</td>\n      <td>127.000000</td>\n      <td>72.825000</td>\n      <td>62.100000</td>\n      <td>328.000000</td>\n    </tr>\n    <tr>\n      <td>50%</td>\n      <td>2009.500000</td>\n      <td>4493.000000</td>\n      <td>7392.000000</td>\n      <td>1939.000000</td>\n      <td>3404.500000</td>\n      <td>3992.000000</td>\n      <td>6070.500000</td>\n      <td>11436.500000</td>\n      <td>5371.000000</td>\n      <td>2.400000</td>\n      <td>...</td>\n      <td>74.200000</td>\n      <td>9.000000</td>\n      <td>71.400000</td>\n      <td>65.800000</td>\n      <td>257.500000</td>\n      <td>10.300000</td>\n      <td>130.000000</td>\n      <td>74.000000</td>\n      <td>63.700000</td>\n      <td>462.000000</td>\n    </tr>\n    <tr>\n      <td>75%</td>\n      <td>2014.000000</td>\n      <td>15848.000000</td>\n      <td>27104.000000</td>\n      <td>7414.000000</td>\n      <td>12259.000000</td>\n      <td>14803.000000</td>\n      <td>23516.000000</td>\n      <td>44564.000000</td>\n      <td>21038.000000</td>\n      <td>2.900000</td>\n      <td>...</td>\n      <td>75.675000</td>\n      <td>9.500000</td>\n      <td>72.300000</td>\n      <td>66.675000</td>\n      <td>690.000000</td>\n      <td>10.900000</td>\n      <td>135.600000</td>\n      <td>74.700000</td>\n      <td>65.100000</td>\n      <td>1241.000000</td>\n    </tr>\n    <tr>\n      <td>max</td>\n      <td>2019.000000</td>\n      <td>58296.000000</td>\n      <td>92916.000000</td>\n      <td>24636.000000</td>\n      <td>42948.000000</td>\n      <td>50772.000000</td>\n      <td>73728.000000</td>\n      <td>139104.000000</td>\n      <td>65580.000000</td>\n      <td>4.600000</td>\n      <td>...</td>\n      <td>79.700000</td>\n      <td>13.200000</td>\n      <td>74.800000</td>\n      <td>71.400000</td>\n      <td>6132.000000</td>\n      <td>14.700000</td>\n      <td>152.800000</td>\n      <td>76.900000</td>\n      <td>71.200000</td>\n      <td>9360.000000</td>\n    </tr>\n  </tbody>\n</table>\n<p>8 rows × 254 columns</p>\n</div>",
      "text/plain": "              Year          YBZY          YBZR          YBZQ          YBZT  \\\ncount   324.000000    322.000000    322.000000    322.000000    322.000000   \nmean   2009.388889  10769.878882  18642.186335   4906.537267   8477.003106   \nstd       5.767691  12388.813285  22382.091688   5795.811994  10105.938143   \nmin    1992.000000   3661.000000   6604.000000   1708.000000   3001.000000   \n25%    2005.000000   4086.250000   6856.250000   1876.250000   3117.500000   \n50%    2009.500000   4493.000000   7392.000000   1939.000000   3404.500000   \n75%    2014.000000  15848.000000  27104.000000   7414.000000  12259.000000   \nmax    2019.000000  58296.000000  92916.000000  24636.000000  42948.000000   \n\n               YBZS          YBZV           YBZU          YBZW        JPC3  \\\ncount    322.000000    322.000000     322.000000    322.000000  218.000000   \nmean   10164.990683  15404.714286   28979.785714  13575.086957    2.550917   \nstd    12281.160677  18046.810211   33833.938916  15791.136565    0.686824   \nmin     3597.000000   5659.000000   10605.000000   4925.000000    1.300000   \n25%     3747.250000   5896.000000   11171.250000   5284.500000    2.100000   \n50%     3992.000000   6070.500000   11436.500000   5371.000000    2.400000   \n75%    14803.000000  23516.000000   44564.000000  21038.000000    2.900000   \nmax    50772.000000  73728.000000  139104.000000  65580.000000    4.600000   \n\n       ...        LF3M        AIYW        LF3T        LF45         YCGN  \\\ncount  ...  322.000000  322.000000  322.000000  322.000000   322.000000   \nmean   ...   74.414907    9.008696   71.050000   66.234161   613.080745   \nstd    ...    1.887397    1.590916    1.731462    2.220511   915.699909   \nmin    ...   70.300000    5.200000   67.500000   60.300000   132.000000   \n25%    ...   73.525000    8.100000   69.725000   64.800000   181.250000   \n50%    ...   74.200000    9.000000   71.400000   65.800000   257.500000   \n75%    ...   75.675000    9.500000   72.300000   66.675000   690.000000   \nmax    ...   79.700000   13.200000   74.800000   71.400000  6132.000000   \n\n             AIYX        DZ36        LF3S        LF44         YCGM  \ncount  322.000000   74.000000  322.000000  322.000000   322.000000  \nmean    10.232609  131.891892   73.667081   63.912422  1079.307453  \nstd      1.724866    7.819413    1.461757    2.689109  1520.665722  \nmin      6.100000  116.800000   70.500000   57.000000   248.000000  \n25%      9.225000  127.000000   72.825000   62.100000   328.000000  \n50%     10.300000  130.000000   74.000000   63.700000   462.000000  \n75%     10.900000  135.600000   74.700000   65.100000  1241.000000  \nmax     14.700000  152.800000   76.900000   71.200000  9360.000000  \n\n[8 rows x 254 columns]"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labor_market_statistics.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "YEAR    27\nAPR     19\nMAR     19\nJUN     19\nJUL     19\nQ2      19\nMAY     19\nQ1      19\nFEB     19\nJAN     19\nQ4      18\nQ3      18\nAUG     18\nSEP     18\nNOV     18\nDEC     18\nOCT     18\nName: Month, dtype: int64"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labor_market_statistics['Month'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Clean the labels, get the names of statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_statistic_name_lfs(column_name):\n",
    "    return column_name.split(':')[1].strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "lm_stats_legend = labor_market_statistics_legend\n",
    "lfs_legend = lm_stats_legend[lm_stats_legend['Title'].apply(lambda x: x[:3] == 'LFS')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "field_names_lfs = lfs_legend['Title'].str.split(':', n=-1, expand=True)\n",
    "lfs_legend['statistic_name'] = field_names_lfs[1].str.strip()\n",
    "lfs_legend['further_info'] = field_names_lfs[2].str.strip()\n",
    "lfs_legend['further_info2'] = field_names_lfs[3].str.strip()\n",
    "lfs_legend['further_info3'] = field_names_lfs[4].str.strip()\n",
    "lfs_legend['further_info4'] = field_names_lfs[5].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Title</th>\n      <th>CDID</th>\n      <th>Unit</th>\n      <th>statistic_name</th>\n      <th>further_info</th>\n      <th>further_info2</th>\n      <th>further_info3</th>\n      <th>further_info4</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>0</td>\n      <td>LFS: Economically active: UK: Male: Aged 50-64...</td>\n      <td>YBZY</td>\n      <td>000's</td>\n      <td>Economically active</td>\n      <td>UK</td>\n      <td>Male</td>\n      <td>Aged 50-64</td>\n      <td>000s</td>\n    </tr>\n    <tr>\n      <td>1</td>\n      <td>LFS: Economically active: UK: All: Aged 25-34:...</td>\n      <td>YBZR</td>\n      <td>000's</td>\n      <td>Economically active</td>\n      <td>UK</td>\n      <td>All</td>\n      <td>Aged 25-34</td>\n      <td>000s</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>LFS: Economically active: UK: Female: Aged 18-...</td>\n      <td>YBZQ</td>\n      <td>000's</td>\n      <td>Economically active</td>\n      <td>UK</td>\n      <td>Female</td>\n      <td>Aged 18-24</td>\n      <td>000s</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>LFS: Economically active: UK: Female: Aged 25-...</td>\n      <td>YBZT</td>\n      <td>000's</td>\n      <td>Economically active</td>\n      <td>UK</td>\n      <td>Female</td>\n      <td>Aged 25-34</td>\n      <td>000s</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>LFS: Economically active: UK: Male: Aged 25-34...</td>\n      <td>YBZS</td>\n      <td>000's</td>\n      <td>Economically active</td>\n      <td>UK</td>\n      <td>Male</td>\n      <td>Aged 25-34</td>\n      <td>000s</td>\n    </tr>\n    <tr>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <td>247</td>\n      <td>LFS: ILO unemployed: Aged 25-34: UK: Male: 000...</td>\n      <td>YCGN</td>\n      <td>000's</td>\n      <td>ILO unemployed</td>\n      <td>Aged 25-34</td>\n      <td>UK</td>\n      <td>Male</td>\n      <td>000s</td>\n    </tr>\n    <tr>\n      <td>248</td>\n      <td>LFS: Econ. inactivity rate: Aged 16-24: Not in...</td>\n      <td>AIYX</td>\n      <td>%</td>\n      <td>Econ. inactivity rate</td>\n      <td>Aged 16-24</td>\n      <td>Not in full-time educ.</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>250</td>\n      <td>LFS: Employment rate: East Midlands: Aged 16-6...</td>\n      <td>LF3S</td>\n      <td>%</td>\n      <td>Employment rate</td>\n      <td>East Midlands</td>\n      <td>Aged 16-64</td>\n      <td>All</td>\n      <td>%</td>\n    </tr>\n    <tr>\n      <td>251</td>\n      <td>LFS: Employment rate: North East: Aged 16-64: ...</td>\n      <td>LF44</td>\n      <td>%</td>\n      <td>Employment rate</td>\n      <td>North East</td>\n      <td>Aged 16-64</td>\n      <td>Female</td>\n      <td>%</td>\n    </tr>\n    <tr>\n      <td>252</td>\n      <td>LFS: ILO unemployed: Aged 25-34: UK: All: 000s...</td>\n      <td>YCGM</td>\n      <td>000's</td>\n      <td>ILO unemployed</td>\n      <td>Aged 25-34</td>\n      <td>UK</td>\n      <td>All</td>\n      <td>000s</td>\n    </tr>\n  </tbody>\n</table>\n<p>236 rows × 8 columns</p>\n</div>",
      "text/plain": "                                                 Title  CDID   Unit  \\\n0    LFS: Economically active: UK: Male: Aged 50-64...  YBZY  000's   \n1    LFS: Economically active: UK: All: Aged 25-34:...  YBZR  000's   \n2    LFS: Economically active: UK: Female: Aged 18-...  YBZQ  000's   \n3    LFS: Economically active: UK: Female: Aged 25-...  YBZT  000's   \n4    LFS: Economically active: UK: Male: Aged 25-34...  YBZS  000's   \n..                                                 ...   ...    ...   \n247  LFS: ILO unemployed: Aged 25-34: UK: Male: 000...  YCGN  000's   \n248  LFS: Econ. inactivity rate: Aged 16-24: Not in...  AIYX      %   \n250  LFS: Employment rate: East Midlands: Aged 16-6...  LF3S      %   \n251  LFS: Employment rate: North East: Aged 16-64: ...  LF44      %   \n252  LFS: ILO unemployed: Aged 25-34: UK: All: 000s...  YCGM  000's   \n\n            statistic_name   further_info           further_info2  \\\n0      Economically active             UK                    Male   \n1      Economically active             UK                     All   \n2      Economically active             UK                  Female   \n3      Economically active             UK                  Female   \n4      Economically active             UK                    Male   \n..                     ...            ...                     ...   \n247         ILO unemployed     Aged 25-34                      UK   \n248  Econ. inactivity rate     Aged 16-24  Not in full-time educ.   \n250        Employment rate  East Midlands              Aged 16-64   \n251        Employment rate     North East              Aged 16-64   \n252         ILO unemployed     Aged 25-34                      UK   \n\n    further_info3 further_info4  \n0      Aged 50-64          000s  \n1      Aged 25-34          000s  \n2      Aged 18-24          000s  \n3      Aged 25-34          000s  \n4      Aged 25-34          000s  \n..            ...           ...  \n247          Male          000s  \n248            UK          Male  \n250           All             %  \n251        Female             %  \n252           All          000s  \n\n[236 rows x 8 columns]"
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lfs_legend"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "Economic activity rate                         27\nEmployment rate                                26\nIn employment                                  24\nTemporary employees                            18\nEcon. inactive                                 18\nUnemployment rate                              18\nPart-time workers                              18\nUnemployed                                     18\nEconomically inactive                          12\nEcon. inactivity rate                          11\nEconomically active                            11\nSelf-employed                                   6\nEmployees                                       6\nTemporary employees as a % of all employees     3\nILO unemployed                                  3\nEconomic inactivity rate                        3\nWorkers with second jobs                        3\nILO unemployment rate                           3\nEcon. inact. rate                               3\nIn Employment                                   2\nILO unemployed up to 6 months                   2\nILO unemployed over 24 months                   1\nName: statistic_name, dtype: int64"
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lfs_legend['statistic_name'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "employed_legend = lfs_legend[lfs_legend['statistic_name'] == 'Employment rate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>statistic_name</th>\n      <th>further_info</th>\n      <th>further_info2</th>\n      <th>further_info3</th>\n      <th>further_info4</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>112</td>\n      <td>Employment rate</td>\n      <td>Aged 16-17</td>\n      <td>Not in full-time education</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>114</td>\n      <td>Employment rate</td>\n      <td>Aged 18-24</td>\n      <td>Not in full-time education</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>115</td>\n      <td>Employment rate</td>\n      <td>Aged 18-24</td>\n      <td>Not in full-time education</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>117</td>\n      <td>Employment rate</td>\n      <td>Aged 16-24</td>\n      <td>Not in full-time education</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>119</td>\n      <td>Employment rate</td>\n      <td>Aged 16-24</td>\n      <td>Not in full-time education</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>121</td>\n      <td>Employment rate</td>\n      <td>Aged 16-17</td>\n      <td>Not in full-time education</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>123</td>\n      <td>Employment rate</td>\n      <td>Aged 16-17</td>\n      <td>Not in full-time education</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>125</td>\n      <td>Employment rate</td>\n      <td>Aged 18-24</td>\n      <td>Not in full-time education</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>128</td>\n      <td>Employment rate</td>\n      <td>Aged 16-24</td>\n      <td>UK</td>\n      <td>Female</td>\n      <td>%</td>\n    </tr>\n    <tr>\n      <td>130</td>\n      <td>Employment rate</td>\n      <td>Aged 16-24</td>\n      <td>UK</td>\n      <td>Male</td>\n      <td>%</td>\n    </tr>\n    <tr>\n      <td>135</td>\n      <td>Employment rate</td>\n      <td>Aged 16-24</td>\n      <td>UK</td>\n      <td>All</td>\n      <td>%</td>\n    </tr>\n    <tr>\n      <td>151</td>\n      <td>Employment rate</td>\n      <td>Aged 16-24</td>\n      <td>In full-time education</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>153</td>\n      <td>Employment rate</td>\n      <td>Aged 16-17</td>\n      <td>In full-time education</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>154</td>\n      <td>Employment rate</td>\n      <td>Aged 18-24</td>\n      <td>In full-time education</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>155</td>\n      <td>Employment rate</td>\n      <td>Aged 18-24</td>\n      <td>In full-time education</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>156</td>\n      <td>Employment rate</td>\n      <td>Aged 16-24</td>\n      <td>In full-time education</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>157</td>\n      <td>Employment rate</td>\n      <td>Aged 16-24</td>\n      <td>In full-time education</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>158</td>\n      <td>Employment rate</td>\n      <td>Aged 16-17</td>\n      <td>In full-time education</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>160</td>\n      <td>Employment rate</td>\n      <td>Aged 18-24</td>\n      <td>In full-time education</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>161</td>\n      <td>Employment rate</td>\n      <td>Aged 16-17</td>\n      <td>In full-time education</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>162</td>\n      <td>Employment rate</td>\n      <td>Aged 16-24</td>\n      <td>Not in full-time education</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>238</td>\n      <td>Employment rate</td>\n      <td>North East</td>\n      <td>Aged 16-64</td>\n      <td>All</td>\n      <td>%</td>\n    </tr>\n    <tr>\n      <td>245</td>\n      <td>Employment rate</td>\n      <td>West Midlands</td>\n      <td>Aged 16-64</td>\n      <td>All</td>\n      <td>%</td>\n    </tr>\n    <tr>\n      <td>246</td>\n      <td>Employment rate</td>\n      <td>North West (GOR)</td>\n      <td>Aged 16-64</td>\n      <td>Female</td>\n      <td>%</td>\n    </tr>\n    <tr>\n      <td>250</td>\n      <td>Employment rate</td>\n      <td>East Midlands</td>\n      <td>Aged 16-64</td>\n      <td>All</td>\n      <td>%</td>\n    </tr>\n    <tr>\n      <td>251</td>\n      <td>Employment rate</td>\n      <td>North East</td>\n      <td>Aged 16-64</td>\n      <td>Female</td>\n      <td>%</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "      statistic_name      further_info               further_info2  \\\n112  Employment rate        Aged 16-17  Not in full-time education   \n114  Employment rate        Aged 18-24  Not in full-time education   \n115  Employment rate        Aged 18-24  Not in full-time education   \n117  Employment rate        Aged 16-24  Not in full-time education   \n119  Employment rate        Aged 16-24  Not in full-time education   \n121  Employment rate        Aged 16-17  Not in full-time education   \n123  Employment rate        Aged 16-17  Not in full-time education   \n125  Employment rate        Aged 18-24  Not in full-time education   \n128  Employment rate        Aged 16-24                          UK   \n130  Employment rate        Aged 16-24                          UK   \n135  Employment rate        Aged 16-24                          UK   \n151  Employment rate        Aged 16-24      In full-time education   \n153  Employment rate        Aged 16-17      In full-time education   \n154  Employment rate        Aged 18-24      In full-time education   \n155  Employment rate        Aged 18-24      In full-time education   \n156  Employment rate        Aged 16-24      In full-time education   \n157  Employment rate        Aged 16-24      In full-time education   \n158  Employment rate        Aged 16-17      In full-time education   \n160  Employment rate        Aged 18-24      In full-time education   \n161  Employment rate        Aged 16-17      In full-time education   \n162  Employment rate        Aged 16-24  Not in full-time education   \n238  Employment rate        North East                  Aged 16-64   \n245  Employment rate     West Midlands                  Aged 16-64   \n246  Employment rate  North West (GOR)                  Aged 16-64   \n250  Employment rate     East Midlands                  Aged 16-64   \n251  Employment rate        North East                  Aged 16-64   \n\n    further_info3 further_info4  \n112            UK        Female  \n114            UK        Female  \n115            UK          Male  \n117            UK          Male  \n119            UK           All  \n121            UK          Male  \n123            UK           All  \n125            UK           All  \n128        Female             %  \n130          Male             %  \n135           All             %  \n151            UK        Female  \n153            UK        Female  \n154            UK        Female  \n155            UK          Male  \n156            UK          Male  \n157            UK           All  \n158            UK          Male  \n160            UK           All  \n161            UK           All  \n162            UK        Female  \n238           All             %  \n245           All             %  \n246        Female             %  \n250           All             %  \n251        Female             %  "
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "employed_legend[['statistic_name', 'further_info', 'further_info2', 'further_info3', 'further_info4']]"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Object `Why` not found.\n"
    }
   ],
   "source": [
    "We can use dataset 3 which splits the labour market into age groups, male/female, unemployment data. Helps us figure out which age group / gender / education level has been most affected by brexit (probably unemployed). For e.g. Age group men 40-50 vs age group 30-40 females educated similar in how they are affected by brexit. Why?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "unemployed_legend = lfs_legend[(lfs_legend['statistic_name'] == 'Unemployment rate') | (lfs_legend['statistic_name'] == 'ILO unemployed')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>statistic_name</th>\n      <th>further_info</th>\n      <th>further_info2</th>\n      <th>further_info3</th>\n      <th>further_info4</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>177</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-24</td>\n      <td>In full-time educ.</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>180</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-17</td>\n      <td>In full-time educ.</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>182</td>\n      <td>Unemployment rate</td>\n      <td>Aged 18-24</td>\n      <td>In full-time educ.</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>184</td>\n      <td>Unemployment rate</td>\n      <td>Aged 18-24</td>\n      <td>In full-time educ.</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>185</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-24</td>\n      <td>In full-time educ.</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>186</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-24</td>\n      <td>In full-time educ.</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>188</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-17</td>\n      <td>In full-time educ.</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>190</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-17</td>\n      <td>In full-time educ.</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>193</td>\n      <td>Unemployment rate</td>\n      <td>Aged 18-24</td>\n      <td>In full-time educ.</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>195</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-24</td>\n      <td>Not in full-time educ.</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>196</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-17</td>\n      <td>Not in full-time educ.</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>197</td>\n      <td>Unemployment rate</td>\n      <td>Aged 18-24</td>\n      <td>Not in full-time educ.</td>\n      <td>UK</td>\n      <td>Female</td>\n    </tr>\n    <tr>\n      <td>198</td>\n      <td>Unemployment rate</td>\n      <td>Aged 18-24</td>\n      <td>Not in full-time educ.</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>200</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-24</td>\n      <td>Not in full-time educ.</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>202</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-24</td>\n      <td>Not in full-time educ.</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>203</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-17</td>\n      <td>Not in full-time educ.</td>\n      <td>UK</td>\n      <td>Male</td>\n    </tr>\n    <tr>\n      <td>205</td>\n      <td>Unemployment rate</td>\n      <td>Aged 16-17</td>\n      <td>Not in full-time educ.</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>206</td>\n      <td>Unemployment rate</td>\n      <td>Aged 18-24</td>\n      <td>Not in full-time educ.</td>\n      <td>UK</td>\n      <td>All</td>\n    </tr>\n    <tr>\n      <td>233</td>\n      <td>ILO unemployed</td>\n      <td>Aged 35-49</td>\n      <td>UK</td>\n      <td>Female</td>\n      <td>000s</td>\n    </tr>\n    <tr>\n      <td>247</td>\n      <td>ILO unemployed</td>\n      <td>Aged 25-34</td>\n      <td>UK</td>\n      <td>Male</td>\n      <td>000s</td>\n    </tr>\n    <tr>\n      <td>252</td>\n      <td>ILO unemployed</td>\n      <td>Aged 25-34</td>\n      <td>UK</td>\n      <td>All</td>\n      <td>000s</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "        statistic_name further_info           further_info2 further_info3  \\\n177  Unemployment rate   Aged 16-24      In full-time educ.            UK   \n180  Unemployment rate   Aged 16-17      In full-time educ.            UK   \n182  Unemployment rate   Aged 18-24      In full-time educ.            UK   \n184  Unemployment rate   Aged 18-24      In full-time educ.            UK   \n185  Unemployment rate   Aged 16-24      In full-time educ.            UK   \n186  Unemployment rate   Aged 16-24      In full-time educ.            UK   \n188  Unemployment rate   Aged 16-17      In full-time educ.            UK   \n190  Unemployment rate   Aged 16-17      In full-time educ.            UK   \n193  Unemployment rate   Aged 18-24      In full-time educ.            UK   \n195  Unemployment rate   Aged 16-24  Not in full-time educ.            UK   \n196  Unemployment rate   Aged 16-17  Not in full-time educ.            UK   \n197  Unemployment rate   Aged 18-24  Not in full-time educ.            UK   \n198  Unemployment rate   Aged 18-24  Not in full-time educ.            UK   \n200  Unemployment rate   Aged 16-24  Not in full-time educ.            UK   \n202  Unemployment rate   Aged 16-24  Not in full-time educ.            UK   \n203  Unemployment rate   Aged 16-17  Not in full-time educ.            UK   \n205  Unemployment rate   Aged 16-17  Not in full-time educ.            UK   \n206  Unemployment rate   Aged 18-24  Not in full-time educ.            UK   \n233     ILO unemployed   Aged 35-49                      UK        Female   \n247     ILO unemployed   Aged 25-34                      UK          Male   \n252     ILO unemployed   Aged 25-34                      UK           All   \n\n    further_info4  \n177        Female  \n180        Female  \n182        Female  \n184          Male  \n185          Male  \n186           All  \n188          Male  \n190           All  \n193           All  \n195        Female  \n196        Female  \n197        Female  \n198          Male  \n200          Male  \n202           All  \n203          Male  \n205           All  \n206           All  \n233          000s  \n247          000s  \n252          000s  "
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unemployed_legend[['statistic_name', 'further_info', 'further_info2', 'further_info3', 'further_info4']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}