2021-05-19_data_process_integrate.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit ('base': conda)",
   "metadata": {
    "interpreter": {
     "hash": "dca0ade3e726a953b501b15e8e990130d2b7799f14cfd9f4271676035ebe5511"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "import pickle\n",
    "import utm\n",
    "import random\n",
    "import matplotlib as mpl\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import DataProcesser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/mid_res/data_detail.pickle', 'rb') as file:\n",
    "    data = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dp = DataProcesser.DataProcesser(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "------------------------ Features in self.data.loc Extracted ------------------------\n"
     ]
    }
   ],
   "source": [
    "dp.process_loc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "------------------------ Features in self.data.wifi(_detail) Extracted ------------------------\n"
     ]
    }
   ],
   "source": [
    "dp.process_wifi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "------------------------ Features in self.data.gps(_detail) Extracted ------------------------\n"
     ]
    }
   ],
   "source": [
    "dp.process_gps()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "------------------------ Features in self.data.cells(_detail) Extracted ------------------------\n"
     ]
    }
   ],
   "source": [
    "dp.process_cells()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                time  accuracy   latitude  longitude   altitude  num_gps  \\\n",
       "0      1490431583000       NaN        NaN        NaN        NaN      4.0   \n",
       "1      1490431584000       NaN        NaN        NaN        NaN      4.0   \n",
       "2      1490431585000       NaN        NaN        NaN        NaN      4.0   \n",
       "3      1490431586000       NaN        NaN        NaN        NaN      4.0   \n",
       "4      1490431587000       NaN        NaN        NaN        NaN      4.0   \n",
       "...              ...       ...        ...        ...        ...      ...   \n",
       "98047  1490861603000      16.0  50.844255  -0.118538  92.085548     14.0   \n",
       "98048  1490861604000      16.0  50.844320  -0.118431  92.227966     14.0   \n",
       "98049  1490861605000      16.0  50.844394  -0.118307  92.347984     14.0   \n",
       "98050  1490861606000      16.0  50.844471  -0.118178  92.438866     14.0   \n",
       "98051  1490861607000      16.0  50.844548  -0.118052  92.545456     14.0   \n",
       "\n",
       "       num_wifi  num_cells  label  \n",
       "0           6.0        NaN      4  \n",
       "1           6.0        NaN      4  \n",
       "2           6.0        NaN      4  \n",
       "3           NaN        NaN      4  \n",
       "4           6.0        NaN      4  \n",
       "...         ...        ...    ...  \n",
       "98047       8.0        NaN      6  \n",
       "98048       8.0        NaN      6  \n",
       "98049       8.0        NaN      6  \n",
       "98050       8.0        NaN      6  \n",
       "98051       8.0        NaN      6  \n",
       "\n",
       "[98052 rows x 9 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>time</th>\n      <th>accuracy</th>\n      <th>latitude</th>\n      <th>longitude</th>\n      <th>altitude</th>\n      <th>num_gps</th>\n      <th>num_wifi</th>\n      <th>num_cells</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1490431583000</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.0</td>\n      <td>6.0</td>\n      <td>NaN</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1490431584000</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.0</td>\n      <td>6.0</td>\n      <td>NaN</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1490431585000</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.0</td>\n      <td>6.0</td>\n      <td>NaN</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1490431586000</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1490431587000</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.0</td>\n      <td>6.0</td>\n      <td>NaN</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>98047</th>\n      <td>1490861603000</td>\n      <td>16.0</td>\n      <td>50.844255</td>\n      <td>-0.118538</td>\n      <td>92.085548</td>\n      <td>14.0</td>\n      <td>8.0</td>\n      <td>NaN</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <th>98048</th>\n      <td>1490861604000</td>\n      <td>16.0</td>\n      <td>50.844320</td>\n      <td>-0.118431</td>\n      <td>92.227966</td>\n      <td>14.0</td>\n      <td>8.0</td>\n      <td>NaN</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <th>98049</th>\n      <td>1490861605000</td>\n      <td>16.0</td>\n      <td>50.844394</td>\n      <td>-0.118307</td>\n      <td>92.347984</td>\n      <td>14.0</td>\n      <td>8.0</td>\n      <td>NaN</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <th>98050</th>\n      <td>1490861606000</td>\n      <td>16.0</td>\n      <td>50.844471</td>\n      <td>-0.118178</td>\n      <td>92.438866</td>\n      <td>14.0</td>\n      <td>8.0</td>\n      <td>NaN</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <th>98051</th>\n      <td>1490861607000</td>\n      <td>16.0</td>\n      <td>50.844548</td>\n      <td>-0.118052</td>\n      <td>92.545456</td>\n      <td>14.0</td>\n      <td>8.0</td>\n      <td>NaN</td>\n      <td>6</td>\n    </tr>\n  </tbody>\n</table>\n<p>98052 rows × 9 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 22
    }
   ],
   "source": [
    "data2.df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "metadata": {},
     "execution_count": 30
    }
   ],
   "source": [
    "isinstance(data.df, pd.core.frame.DataFrame)"
   ]
  },
  {
   "source": [
    "data.wifi_detail"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 43,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                  time              bssid                  ssid rssi  freq  \\\n",
       "0        1490430923000  c0:05:c2:29:40:ff             VM8236218  -56  5220   \n",
       "1        1490430923000  c4:04:15:e4:5b:30           VM201213-2G  -70  2437   \n",
       "2        1490430923000  a0:63:91:a2:5a:50       VM702835-2G_EXT  -85  2462   \n",
       "3        1490430923000  c0:05:c2:29:40:f9             VM8236218  -55  2412   \n",
       "4        1490430923000  98:e7:f5:b9:3f:14        TALKTALKB93F0E  -65  2432   \n",
       "...                ...                ...                   ...  ...   ...   \n",
       "1714352  1492602350000  8c:eb:c6:e5:61:49  MTN-MobileWiFi-E5573  -47  2412   \n",
       "1714353  1492602350000  00:20:c7:83:8e:0e           BTHub5-WT6X  -88  2462   \n",
       "1714354  1492602350000  00:fe:c8:cf:95:bf               O2 Wifi  -75  5220   \n",
       "1714355  1492602350000  00:fe:c8:da:d8:cf               O2 Wifi  -82  5320   \n",
       "1714356  1492602350000  00:fe:c8:cf:95:b0               O2 Wifi  -81  2412   \n",
       "\n",
       "                                                       cap  wifi_rssi  \\\n",
       "0                           [WPA2-PSK-CCMP+TKIP][WPS][ESS]        -56   \n",
       "1        [WPA-PSK-CCMP+TKIP][WPA2-PSK-CCMP+TKIP][WPS][ESS]        -70   \n",
       "2        [WPA-PSK-CCMP+TKIP][WPA2-PSK-CCMP+TKIP][WPS][ESS]        -85   \n",
       "3        [WPA-PSK-CCMP+TKIP][WPA2-PSK-CCMP+TKIP][WPS][ESS]        -55   \n",
       "4        [WPA-PSK-CCMP+TKIP][WPA2-PSK-CCMP+TKIP][WPS][ESS]        -65   \n",
       "...                                                    ...        ...   \n",
       "1714352                               [WPA2-PSK-CCMP][ESS]        -47   \n",
       "1714353                          [WPA2-PSK-CCMP][WPS][ESS]        -88   \n",
       "1714354                                              [ESS]        -75   \n",
       "1714355                                              [ESS]        -82   \n",
       "1714356                                              [ESS]        -81   \n",
       "\n",
       "         wifi_freq  \n",
       "0              5.0  \n",
       "1              2.4  \n",
       "2              2.4  \n",
       "3              2.4  \n",
       "4              2.4  \n",
       "...            ...  \n",
       "1714352        2.4  \n",
       "1714353        2.4  \n",
       "1714354        5.0  \n",
       "1714355        5.0  \n",
       "1714356        2.4  \n",
       "\n",
       "[1714357 rows x 8 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>time</th>\n      <th>bssid</th>\n      <th>ssid</th>\n      <th>rssi</th>\n      <th>freq</th>\n      <th>cap</th>\n      <th>wifi_rssi</th>\n      <th>wifi_freq</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1490430923000</td>\n      <td>c0:05:c2:29:40:ff</td>\n      <td>VM8236218</td>\n      <td>-56</td>\n      <td>5220</td>\n      <td>[WPA2-PSK-CCMP+TKIP][WPS][ESS]</td>\n      <td>-56</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1490430923000</td>\n      <td>c4:04:15:e4:5b:30</td>\n      <td>VM201213-2G</td>\n      <td>-70</td>\n      <td>2437</td>\n      <td>[WPA-PSK-CCMP+TKIP][WPA2-PSK-CCMP+TKIP][WPS][ESS]</td>\n      <td>-70</td>\n      <td>2.4</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1490430923000</td>\n      <td>a0:63:91:a2:5a:50</td>\n      <td>VM702835-2G_EXT</td>\n      <td>-85</td>\n      <td>2462</td>\n      <td>[WPA-PSK-CCMP+TKIP][WPA2-PSK-CCMP+TKIP][WPS][ESS]</td>\n      <td>-85</td>\n      <td>2.4</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1490430923000</td>\n      <td>c0:05:c2:29:40:f9</td>\n      <td>VM8236218</td>\n      <td>-55</td>\n      <td>2412</td>\n      <td>[WPA-PSK-CCMP+TKIP][WPA2-PSK-CCMP+TKIP][WPS][ESS]</td>\n      <td>-55</td>\n      <td>2.4</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1490430923000</td>\n      <td>98:e7:f5:b9:3f:14</td>\n      <td>TALKTALKB93F0E</td>\n      <td>-65</td>\n      <td>2432</td>\n      <td>[WPA-PSK-CCMP+TKIP][WPA2-PSK-CCMP+TKIP][WPS][ESS]</td>\n      <td>-65</td>\n      <td>2.4</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1714352</th>\n      <td>1492602350000</td>\n      <td>8c:eb:c6:e5:61:49</td>\n      <td>MTN-MobileWiFi-E5573</td>\n      <td>-47</td>\n      <td>2412</td>\n      <td>[WPA2-PSK-CCMP][ESS]</td>\n      <td>-47</td>\n      <td>2.4</td>\n    </tr>\n    <tr>\n      <th>1714353</th>\n      <td>1492602350000</td>\n      <td>00:20:c7:83:8e:0e</td>\n      <td>BTHub5-WT6X</td>\n      <td>-88</td>\n      <td>2462</td>\n      <td>[WPA2-PSK-CCMP][WPS][ESS]</td>\n      <td>-88</td>\n      <td>2.4</td>\n    </tr>\n    <tr>\n      <th>1714354</th>\n      <td>1492602350000</td>\n      <td>00:fe:c8:cf:95:bf</td>\n      <td>O2 Wifi</td>\n      <td>-75</td>\n      <td>5220</td>\n      <td>[ESS]</td>\n      <td>-75</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>1714355</th>\n      <td>1492602350000</td>\n      <td>00:fe:c8:da:d8:cf</td>\n      <td>O2 Wifi</td>\n      <td>-82</td>\n      <td>5320</td>\n      <td>[ESS]</td>\n      <td>-82</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>1714356</th>\n      <td>1492602350000</td>\n      <td>00:fe:c8:cf:95:b0</td>\n      <td>O2 Wifi</td>\n      <td>-81</td>\n      <td>2412</td>\n      <td>[ESS]</td>\n      <td>-81</td>\n      <td>2.4</td>\n    </tr>\n  </tbody>\n</table>\n<p>1714357 rows × 8 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 43
    }
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "------------------------ Features in self.data.wifi(_detail) Extracted ------------------------\nNew Feature Added: {'wifi_rssi_max_x', 'wifi_rssi_mode_x', 'wifi_rssi_std_y', 'wifi_rssi_min_y', 'wifi_freq_5ratio', 'wifi_rssi_std_x', 'wifi_rssi_mean_y', 'wifi_rssi_mode_y', 'wifi_rssi_max_y', 'wifi_rssi_mean_x', 'wifi_rssi_min_x'}\n"
     ]
    }
   ],
   "source": [
    "dp.process_wifi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}