activity_tracker.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import datetime as dt\n",
    "from datetime import *\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cf = pd.read_csv('classification.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RECTYPE</th>\n",
       "      <th>CASEID</th>\n",
       "      <th>METRO</th>\n",
       "      <th>FULLPART</th>\n",
       "      <th>ACTLINE</th>\n",
       "      <th>ACTIVITY</th>\n",
       "      <th>START</th>\n",
       "      <th>STOP</th>\n",
       "      <th>activity</th>\n",
       "      <th>location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>20170101170090</td>\n",
       "      <td>Nonmetropolitan</td>\n",
       "      <td>Full time</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Travel related to working</td>\n",
       "      <td>2019-05-20 04:00:00</td>\n",
       "      <td>2019-05-20 05:45:00</td>\n",
       "      <td>Travel related to working</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>20170101170090</td>\n",
       "      <td>Nonmetropolitan</td>\n",
       "      <td>Full time</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Travel related to working</td>\n",
       "      <td>2019-05-20 14:00:00</td>\n",
       "      <td>2019-05-20 15:45:00</td>\n",
       "      <td>Travel related to working</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>20170101170572</td>\n",
       "      <td>Nonmetropolitan</td>\n",
       "      <td>Full time</td>\n",
       "      <td>6.0</td>\n",
       "      <td>Travel related to working</td>\n",
       "      <td>2019-05-20 11:30:00</td>\n",
       "      <td>2019-05-20 11:45:00</td>\n",
       "      <td>Travel related to working</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>20170101170572</td>\n",
       "      <td>Nonmetropolitan</td>\n",
       "      <td>Full time</td>\n",
       "      <td>13.0</td>\n",
       "      <td>Travel related to working</td>\n",
       "      <td>2019-05-20 21:00:00</td>\n",
       "      <td>2019-05-20 21:15:00</td>\n",
       "      <td>Travel related to working</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>20170101170726</td>\n",
       "      <td>Nonmetropolitan</td>\n",
       "      <td>Full time</td>\n",
       "      <td>13.0</td>\n",
       "      <td>Travel related to working</td>\n",
       "      <td>2019-05-20 16:26:00</td>\n",
       "      <td>2019-05-20 16:46:00</td>\n",
       "      <td>Travel related to working</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   RECTYPE          CASEID            METRO   FULLPART  ACTLINE  \\\n",
       "0        3  20170101170090  Nonmetropolitan  Full time      1.0   \n",
       "1        3  20170101170090  Nonmetropolitan  Full time      3.0   \n",
       "2        3  20170101170572  Nonmetropolitan  Full time      6.0   \n",
       "3        3  20170101170572  Nonmetropolitan  Full time     13.0   \n",
       "4        3  20170101170726  Nonmetropolitan  Full time     13.0   \n",
       "\n",
       "                    ACTIVITY               START                STOP  \\\n",
       "0  Travel related to working 2019-05-20 04:00:00 2019-05-20 05:45:00   \n",
       "1  Travel related to working 2019-05-20 14:00:00 2019-05-20 15:45:00   \n",
       "2  Travel related to working 2019-05-20 11:30:00 2019-05-20 11:45:00   \n",
       "3  Travel related to working 2019-05-20 21:00:00 2019-05-20 21:15:00   \n",
       "4  Travel related to working 2019-05-20 16:26:00 2019-05-20 16:46:00   \n",
       "\n",
       "                    activity location  \n",
       "0  Travel related to working        P  \n",
       "1  Travel related to working        P  \n",
       "2  Travel related to working        P  \n",
       "3  Travel related to working        P  \n",
       "4  Travel related to working        P  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('b.csv')\n",
    "df = df[(df.METRO == 'Nonmetropolitan') &  (df.FULLPART != 'Part time')].copy()\n",
    "#df = df[(df.METRO == 'Nonmetropolitan') &  (df.FULLPART == 'Full time') & (df.ACTIVITY == 'Work, main job')]\n",
    "df['START'] = pd.to_datetime(df['START'])\n",
    "\n",
    "df['STOP'] = pd.to_datetime(df['STOP'])\n",
    "\n",
    "df = df.merge(cf, left_on='ACTIVITY', right_on='activity').copy()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "interval = 30*60\n",
    "midnight  = datetime.combine(datetime.today(), datetime.min.time())\n",
    "end_day = datetime.combine(datetime.today(), datetime.max.time())\n",
    "\n",
    "def a(s):\n",
    "    if s.START > s.STOP:\n",
    "        r = (end_day - datetime.combine(datetime.today(), s.START.time())) + (s.STOP- midnight)       \n",
    "    else:\n",
    "        r = s.STOP - s.START\n",
    "    return r\n",
    "        \n",
    "\n",
    "df['duration'] = df.apply(a,axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(m):\n",
    "    row_sums = m.sum(axis=1)\n",
    "    return m / row_sums[:, np.newaxis]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "t1 = datetime.combine(datetime.today(), time(19,0,0))\n",
    "t2 = datetime.combine(datetime.today(), time(6,30,0))\n",
    "t3 = datetime.combine(datetime.today(), time(9,30,0))\n",
    "t4 = datetime.combine(datetime.today(), time(16,30,0))\n",
    "\n",
    "\n",
    "# change this to != 'Full time' for unemployed transition matrix\n",
    "df = df[(df.FULLPART == 'Full time')].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "interval = 30*60\n",
    "A_times = [(x-interval, x-1) for x in range(int(19*60*60), (int(24*60*60)), interval)[1:]] + \\\n",
    "           [(x-interval, x-1) for x in range(0, (int(6.5*60*60)), interval)[1:]]\n",
    "B_times = [(x-interval, x-1) for x in range(int(6.5*60*60), (int(9.5*60*60))+interval, interval)[1:]]\n",
    "C_times = [(x-interval, x-1) for x in range(int(16.5*60*60), (int(19*60*60)), interval)[1:]]\n",
    "D_times = [(x-interval, x-1) for x in range(int(9.5*60*60), (int(16.5*60*60)), interval)[1:]]\n",
    "\n",
    "#A_times, B_times, C_times, D_times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_transit_matrix(m, df, times):\n",
    "    midnight  = datetime.combine(datetime.today(), datetime.min.time())\n",
    "    k = {'H':0, 'W':1, 'P':2}\n",
    "    \n",
    "    \n",
    "    for case in df.CASEID.unique(): #3[20170101170090]:#\n",
    "       \n",
    "        b = df[df.CASEID == case].copy()\n",
    "        c_loc = None\n",
    "        ldd = set()\n",
    "        \n",
    "        status = np.zeros((3, 3),dtype=int)\n",
    "        \n",
    "        \n",
    "        for start,end in times:\n",
    "            \n",
    "            start_time = (midnight + timedelta(seconds= start))\n",
    "            end_time = (midnight + timedelta(seconds=end))\n",
    "        \n",
    "            nature_mask = (b.START > b.STOP)\n",
    "            true_mask = (b.STOP > start_time) | (b.START < end_time)\n",
    "            false_mask = (b.START < end_time) & (b.STOP > start_time)\n",
    "            \n",
    "            \n",
    "            #print(start_time, end_time)\n",
    "\n",
    "          \n",
    "            final =  ((nature_mask & true_mask) | (~nature_mask & false_mask))\n",
    "            #print(false)\n",
    "\n",
    "            b = b[final].copy()\n",
    "            r = b.sort_values(['ACTLINE'], ascending=False)\n",
    "\n",
    "\n",
    "            if len(r.location)>0:\n",
    "                loc = list(r.location)\n",
    "                \n",
    "                if c_loc == None:\n",
    "                    c_loc = loc[0]\n",
    "                    loc = loc[1:]\n",
    "                    ldd.add(c_loc)\n",
    "            \n",
    "                for l in loc:\n",
    "                    ldd.add(l)\n",
    "                    if c_loc  != l and status[k[c_loc]][k[l]] == 0:\n",
    "\n",
    "                        m[k[c_loc]][k[l]] +=1\n",
    "                        status[k[c_loc]][k[l]] += 1\n",
    "                    #c_loc = l\n",
    "                    c_loc = l\n",
    "                    \n",
    "        \n",
    "        t = 0\n",
    "        for i in range(len(status)):\n",
    "            t += sum(status[i])\n",
    "            \n",
    "        if t == 0 and c_loc != None:\n",
    "            m[k[c_loc]][k[c_loc]] +=1\n",
    "            \n",
    "            if len(ldd) > 1:\n",
    "                print(len(ldd))\n",
    "            \n",
    "            #pass\n",
    "        \n",
    "               # c_loc = l\n",
    "\n",
    "    return m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A is done!\n",
      "B is done!\n",
      "C is done!\n",
      "D is done!\n"
     ]
    }
   ],
   "source": [
    "A = np.zeros((3, 3),dtype=int)\n",
    "B = np.zeros((3, 3),dtype=int)\n",
    "C = np.zeros((3, 3),dtype=int)\n",
    "D = np.zeros((3, 3),dtype=int)\n",
    "\n",
    "\n",
    "for tb in [1, 2, 3, 4]:\n",
    "    if tb ==1:\n",
    "        A = get_transit_matrix(A, df, A_times)\n",
    "        print('A is done!')\n",
    "        \n",
    "    elif (tb == 2):\n",
    "        B = get_transit_matrix(B, df, B_times)\n",
    "        print('B is done!')\n",
    "    \n",
    "    elif (tb == 3):\n",
    "        C = get_transit_matrix(C, df, C_times)\n",
    "        print('C is done!')\n",
    "                \n",
    "    elif (tb == 4):\n",
    "        D = get_transit_matrix(D, df, D_times)\n",
    "        print('D is done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[410,   9,  61],\n",
       "       [ 10,  57,   3],\n",
       "       [ 60,   5,  89]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[435,  40,  31],\n",
       "       [ 23,  83,  29],\n",
       "       [ 42,  15,  26]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[270,   9,  58],\n",
       "       [  7, 146,  13],\n",
       "       [ 58,  12, 136]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[241,  25,  35],\n",
       "       [ 17, 243,  10],\n",
       "       [ 37,   7,  79]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "{'H':0, 'W':1, 'P':2}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.85416667, 0.01875   , 0.12708333],\n",
       "       [0.14285714, 0.81428571, 0.04285714],\n",
       "       [0.38961039, 0.03246753, 0.57792208]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalize(A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.85968379, 0.07905138, 0.06126482],\n",
       "       [0.17037037, 0.61481481, 0.21481481],\n",
       "       [0.5060241 , 0.18072289, 0.31325301]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalize(B)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.80118694, 0.02670623, 0.17210682],\n",
       "       [0.04216867, 0.87951807, 0.07831325],\n",
       "       [0.2815534 , 0.05825243, 0.66019417]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalize(C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.80066445, 0.08305648, 0.11627907],\n",
       "       [0.06296296, 0.9       , 0.03703704],\n",
       "       [0.30081301, 0.05691057, 0.64227642]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalize(D)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}