03_transform_ts_data_into_features_and_targets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pickup_hour</th>\n",
       "      <th>rides</th>\n",
       "      <th>pickup_location_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-01-01 00:00:00</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-01-01 01:00:00</td>\n",
       "      <td>15</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-01-01 02:00:00</td>\n",
       "      <td>26</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-01-01 03:00:00</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-01-01 04:00:00</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191203</th>\n",
       "      <td>2022-01-31 19:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191204</th>\n",
       "      <td>2022-01-31 20:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191205</th>\n",
       "      <td>2022-01-31 21:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191206</th>\n",
       "      <td>2022-01-31 22:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191207</th>\n",
       "      <td>2022-01-31 23:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>191208 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               pickup_hour  rides  pickup_location_id\n",
       "0      2022-01-01 00:00:00     11                   4\n",
       "1      2022-01-01 01:00:00     15                   4\n",
       "2      2022-01-01 02:00:00     26                   4\n",
       "3      2022-01-01 03:00:00      8                   4\n",
       "4      2022-01-01 04:00:00      9                   4\n",
       "...                    ...    ...                 ...\n",
       "191203 2022-01-31 19:00:00      0                 176\n",
       "191204 2022-01-31 20:00:00      0                 176\n",
       "191205 2022-01-31 21:00:00      0                 176\n",
       "191206 2022-01-31 22:00:00      0                 176\n",
       "191207 2022-01-31 23:00:00      0                 176\n",
       "\n",
       "[191208 rows x 3 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "ts_data = pd.read_parquet('data/transformed/ts_data_2022_01.parquet')\n",
    "ts_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pickup_hour</th>\n",
       "      <th>rides</th>\n",
       "      <th>pickup_location_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-01-01 00:00:00</td>\n",
       "      <td>97</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-01-01 01:00:00</td>\n",
       "      <td>60</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-01-01 02:00:00</td>\n",
       "      <td>22</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-01-01 03:00:00</td>\n",
       "      <td>8</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-01-01 04:00:00</td>\n",
       "      <td>6</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2022-01-01 05:00:00</td>\n",
       "      <td>5</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2022-01-01 06:00:00</td>\n",
       "      <td>3</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2022-01-01 07:00:00</td>\n",
       "      <td>10</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2022-01-01 08:00:00</td>\n",
       "      <td>7</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2022-01-01 09:00:00</td>\n",
       "      <td>19</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2022-01-01 10:00:00</td>\n",
       "      <td>24</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2022-01-01 11:00:00</td>\n",
       "      <td>39</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2022-01-01 12:00:00</td>\n",
       "      <td>35</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2022-01-01 13:00:00</td>\n",
       "      <td>77</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2022-01-01 14:00:00</td>\n",
       "      <td>70</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           pickup_hour  rides  pickup_location_id\n",
       "0  2022-01-01 00:00:00     97                  43\n",
       "1  2022-01-01 01:00:00     60                  43\n",
       "2  2022-01-01 02:00:00     22                  43\n",
       "3  2022-01-01 03:00:00      8                  43\n",
       "4  2022-01-01 04:00:00      6                  43\n",
       "5  2022-01-01 05:00:00      5                  43\n",
       "6  2022-01-01 06:00:00      3                  43\n",
       "7  2022-01-01 07:00:00     10                  43\n",
       "8  2022-01-01 08:00:00      7                  43\n",
       "9  2022-01-01 09:00:00     19                  43\n",
       "10 2022-01-01 10:00:00     24                  43\n",
       "11 2022-01-01 11:00:00     39                  43\n",
       "12 2022-01-01 12:00:00     35                  43\n",
       "13 2022-01-01 13:00:00     77                  43\n",
       "14 2022-01-01 14:00:00     70                  43"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_data_one_location = ts_data.loc[ts_data.pickup_location_id == 43, :].reset_index(drop=True)\n",
    "ts_data_one_location.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cutoff_indices(\n",
    "    data: pd.DataFrame,\n",
    "    n_features: int,\n",
    "    step_size: int\n",
    "    ) -> list:\n",
    "\n",
    "        stop_position = len(data) - 1\n",
    "        \n",
    "        # Start the first sub-sequence at index position 0\n",
    "        subseq_first_idx = 0\n",
    "        subseq_mid_idx = n_features\n",
    "        subseq_last_idx = n_features + 1\n",
    "        indices = []\n",
    "        \n",
    "        while subseq_last_idx <= stop_position:\n",
    "            indices.append((subseq_first_idx, subseq_mid_idx, subseq_last_idx))\n",
    "            \n",
    "            subseq_first_idx += step_size\n",
    "            subseq_mid_idx += step_size\n",
    "            subseq_last_idx += step_size\n",
    "\n",
    "        return indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 24, 25), (1, 25, 26), (2, 26, 27), (3, 27, 28), (4, 28, 29)]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_features = 24\n",
    "step_size = 1\n",
    "\n",
    "indices = get_cutoff_indices(\n",
    "    ts_data_one_location,\n",
    "    n_features,\n",
    "    step_size\n",
    ")\n",
    "indices[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "n_examples = len(indices)\n",
    "x = np.ndarray(shape=(n_examples, n_features), dtype=np.float32)\n",
    "y = np.ndarray(shape=(n_examples), dtype=np.float32)\n",
    "pickup_hours = []\n",
    "\n",
    "for i, idx in enumerate(indices):\n",
    "    x[i, :] = ts_data_one_location.iloc[idx[0]:idx[1]]['rides'].values\n",
    "    y[i] = ts_data_one_location.iloc[idx[1]:idx[2]]['rides'].values\n",
    "    pickup_hours.append(ts_data_one_location.iloc[idx[1]]['pickup_hour'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x.shape=(719, 24)\n",
      "x=array([[ 97.,  60.,  22., ...,  16.,  18.,   6.],\n",
      "       [ 60.,  22.,   8., ...,  18.,   6.,   3.],\n",
      "       [ 22.,   8.,   6., ...,   6.,   3.,   1.],\n",
      "       ...,\n",
      "       [ 28.,  16.,  13., ..., 102.,  66.,  61.],\n",
      "       [ 16.,  13.,   8., ...,  66.,  61.,  73.],\n",
      "       [ 13.,   8.,   1., ...,  61.,  73.,  33.]], dtype=float32)\n",
      "pickup_hours[:5]=[Timestamp('2022-01-02 00:00:00'), Timestamp('2022-01-02 01:00:00'), Timestamp('2022-01-02 02:00:00'), Timestamp('2022-01-02 03:00:00'), Timestamp('2022-01-02 04:00:00')]\n"
     ]
    }
   ],
   "source": [
    "print(f'{x.shape=}')\n",
    "print(f'{x=}')\n",
    "print(f'{pickup_hours[:5]=}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rides_previous_24_hour</th>\n",
       "      <th>rides_previous_23_hour</th>\n",
       "      <th>rides_previous_22_hour</th>\n",
       "      <th>rides_previous_21_hour</th>\n",
       "      <th>rides_previous_20_hour</th>\n",
       "      <th>rides_previous_19_hour</th>\n",
       "      <th>rides_previous_18_hour</th>\n",
       "      <th>rides_previous_17_hour</th>\n",
       "      <th>rides_previous_16_hour</th>\n",
       "      <th>rides_previous_15_hour</th>\n",
       "      <th>...</th>\n",
       "      <th>rides_previous_10_hour</th>\n",
       "      <th>rides_previous_9_hour</th>\n",
       "      <th>rides_previous_8_hour</th>\n",
       "      <th>rides_previous_7_hour</th>\n",
       "      <th>rides_previous_6_hour</th>\n",
       "      <th>rides_previous_5_hour</th>\n",
       "      <th>rides_previous_4_hour</th>\n",
       "      <th>rides_previous_3_hour</th>\n",
       "      <th>rides_previous_2_hour</th>\n",
       "      <th>rides_previous_1_hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>97.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>...</td>\n",
       "      <td>70.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>60.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>...</td>\n",
       "      <td>94.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>22.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>...</td>\n",
       "      <td>87.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>...</td>\n",
       "      <td>73.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>...</td>\n",
       "      <td>34.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>714</th>\n",
       "      <td>52.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>78.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>106.0</td>\n",
       "      <td>147.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>102.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>715</th>\n",
       "      <td>36.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>74.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>106.0</td>\n",
       "      <td>147.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>66.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>716</th>\n",
       "      <td>28.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>66.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>106.0</td>\n",
       "      <td>147.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>61.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>717</th>\n",
       "      <td>16.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>...</td>\n",
       "      <td>91.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>106.0</td>\n",
       "      <td>147.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>73.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718</th>\n",
       "      <td>13.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>...</td>\n",
       "      <td>117.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>106.0</td>\n",
       "      <td>147.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>33.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>719 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     rides_previous_24_hour  rides_previous_23_hour  rides_previous_22_hour  \\\n",
       "0                      97.0                    60.0                    22.0   \n",
       "1                      60.0                    22.0                     8.0   \n",
       "2                      22.0                     8.0                     6.0   \n",
       "3                       8.0                     6.0                     5.0   \n",
       "4                       6.0                     5.0                     3.0   \n",
       "..                      ...                     ...                     ...   \n",
       "714                    52.0                    36.0                    28.0   \n",
       "715                    36.0                    28.0                    16.0   \n",
       "716                    28.0                    16.0                    13.0   \n",
       "717                    16.0                    13.0                     8.0   \n",
       "718                    13.0                     8.0                     1.0   \n",
       "\n",
       "     rides_previous_21_hour  rides_previous_20_hour  rides_previous_19_hour  \\\n",
       "0                       8.0                     6.0                     5.0   \n",
       "1                       6.0                     5.0                     3.0   \n",
       "2                       5.0                     3.0                    10.0   \n",
       "3                       3.0                    10.0                     7.0   \n",
       "4                      10.0                     7.0                    19.0   \n",
       "..                      ...                     ...                     ...   \n",
       "714                    16.0                    13.0                     8.0   \n",
       "715                    13.0                     8.0                     1.0   \n",
       "716                     8.0                     1.0                     1.0   \n",
       "717                     1.0                     1.0                     2.0   \n",
       "718                     1.0                     2.0                     1.0   \n",
       "\n",
       "     rides_previous_18_hour  rides_previous_17_hour  rides_previous_16_hour  \\\n",
       "0                       3.0                    10.0                     7.0   \n",
       "1                      10.0                     7.0                    19.0   \n",
       "2                       7.0                    19.0                    24.0   \n",
       "3                      19.0                    24.0                    39.0   \n",
       "4                      24.0                    39.0                    35.0   \n",
       "..                      ...                     ...                     ...   \n",
       "714                     1.0                     1.0                     2.0   \n",
       "715                     1.0                     2.0                     1.0   \n",
       "716                     2.0                     1.0                     1.0   \n",
       "717                     1.0                     1.0                     4.0   \n",
       "718                     1.0                     4.0                     9.0   \n",
       "\n",
       "     rides_previous_15_hour  ...  rides_previous_10_hour  \\\n",
       "0                      19.0  ...                    70.0   \n",
       "1                      24.0  ...                    94.0   \n",
       "2                      39.0  ...                    87.0   \n",
       "3                      35.0  ...                    73.0   \n",
       "4                      77.0  ...                    34.0   \n",
       "..                      ...  ...                     ...   \n",
       "714                     1.0  ...                    78.0   \n",
       "715                     1.0  ...                    74.0   \n",
       "716                     4.0  ...                    66.0   \n",
       "717                     9.0  ...                    91.0   \n",
       "718                    53.0  ...                   117.0   \n",
       "\n",
       "     rides_previous_9_hour  rides_previous_8_hour  rides_previous_7_hour  \\\n",
       "0                     94.0                   87.0                   73.0   \n",
       "1                     87.0                   73.0                   34.0   \n",
       "2                     73.0                   34.0                   32.0   \n",
       "3                     34.0                   32.0                   22.0   \n",
       "4                     32.0                   22.0                   16.0   \n",
       "..                     ...                    ...                    ...   \n",
       "714                   74.0                   66.0                   91.0   \n",
       "715                   66.0                   91.0                  117.0   \n",
       "716                   91.0                  117.0                  100.0   \n",
       "717                  117.0                  100.0                  106.0   \n",
       "718                  100.0                  106.0                  147.0   \n",
       "\n",
       "     rides_previous_6_hour  rides_previous_5_hour  rides_previous_4_hour  \\\n",
       "0                     34.0                   32.0                   22.0   \n",
       "1                     32.0                   22.0                   16.0   \n",
       "2                     22.0                   16.0                   18.0   \n",
       "3                     16.0                   18.0                    6.0   \n",
       "4                     18.0                    6.0                    3.0   \n",
       "..                     ...                    ...                    ...   \n",
       "714                  117.0                  100.0                  106.0   \n",
       "715                  100.0                  106.0                  147.0   \n",
       "716                  106.0                  147.0                  121.0   \n",
       "717                  147.0                  121.0                  102.0   \n",
       "718                  121.0                  102.0                   66.0   \n",
       "\n",
       "     rides_previous_3_hour  rides_previous_2_hour  rides_previous_1_hour  \n",
       "0                     16.0                   18.0                    6.0  \n",
       "1                     18.0                    6.0                    3.0  \n",
       "2                      6.0                    3.0                    1.0  \n",
       "3                      3.0                    1.0                    1.0  \n",
       "4                      1.0                    1.0                    0.0  \n",
       "..                     ...                    ...                    ...  \n",
       "714                  147.0                  121.0                  102.0  \n",
       "715                  121.0                  102.0                   66.0  \n",
       "716                  102.0                   66.0                   61.0  \n",
       "717                   66.0                   61.0                   73.0  \n",
       "718                   61.0                   73.0                   33.0  \n",
       "\n",
       "[719 rows x 24 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_one_location = pd.DataFrame(\n",
    "    x,\n",
    "    columns=[f'rides_previous_{i+1}_hour' for i in reversed(range(n_features))]\n",
    ")\n",
    "features_one_location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target_rides_next_hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>714</th>\n",
       "      <td>66.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>715</th>\n",
       "      <td>61.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>716</th>\n",
       "      <td>73.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>717</th>\n",
       "      <td>33.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718</th>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>719 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     target_rides_next_hour\n",
       "0                       3.0\n",
       "1                       1.0\n",
       "2                       1.0\n",
       "3                       0.0\n",
       "4                       0.0\n",
       "..                      ...\n",
       "714                    66.0\n",
       "715                    61.0\n",
       "716                    73.0\n",
       "717                    33.0\n",
       "718                    21.0\n",
       "\n",
       "[719 rows x 1 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets_one_location = pd.DataFrame(y, columns=[f'target_rides_next_hour'])\n",
    "targets_one_location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def transform_ts_data_into_features_and_target(\n",
    "    ts_data: pd.DataFrame,\n",
    "    input_seq_len: int,\n",
    "    step_size: int\n",
    ") -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Slices and transposes data from time-series format into a (features, target)\n",
    "    format that we can use to train Supervised ML models\n",
    "    \"\"\"\n",
    "    assert set(ts_data.columns) == {'pickup_hour', 'rides', 'pickup_location_id'}\n",
    "\n",
    "    location_ids = ts_data['pickup_location_id'].unique()\n",
    "    features = pd.DataFrame()\n",
    "    targets = pd.DataFrame()\n",
    "    \n",
    "    for location_id in tqdm(location_ids):\n",
    "        \n",
    "        # keep only ts data for this `location_id`\n",
    "        ts_data_one_location = ts_data.loc[\n",
    "            ts_data.pickup_location_id == location_id, \n",
    "            ['pickup_hour', 'rides']\n",
    "        ]\n",
    "\n",
    "        # pre-compute cutoff indices to split dataframe rows\n",
    "        indices = get_cutoff_indices(\n",
    "            ts_data_one_location,\n",
    "            input_seq_len,\n",
    "            step_size\n",
    "        )\n",
    "\n",
    "        # slice and transpose data into numpy arrays for features and targets\n",
    "        n_examples = len(indices)\n",
    "        x = np.ndarray(shape=(n_examples, input_seq_len), dtype=np.float32)\n",
    "        y = np.ndarray(shape=(n_examples), dtype=np.float32)\n",
    "        pickup_hours = []\n",
    "        for i, idx in enumerate(indices):\n",
    "            x[i, :] = ts_data_one_location.iloc[idx[0]:idx[1]]['rides'].values\n",
    "            y[i] = ts_data_one_location.iloc[idx[1]:idx[2]]['rides'].values\n",
    "            pickup_hours.append(ts_data_one_location.iloc[idx[1]]['pickup_hour'])\n",
    "\n",
    "        # numpy -> pandas\n",
    "        features_one_location = pd.DataFrame(\n",
    "            x,\n",
    "            columns=[f'rides_previous_{i+1}_hour' for i in reversed(range(input_seq_len))]\n",
    "        )\n",
    "        features_one_location['pickup_hour'] = pickup_hours\n",
    "        features_one_location['pickup_location_id'] = location_id\n",
    "\n",
    "        # numpy -> pandas\n",
    "        targets_one_location = pd.DataFrame(y, columns=[f'target_rides_next_hour'])\n",
    "\n",
    "        # concatenate results\n",
    "        features = pd.concat([features, features_one_location])\n",
    "        targets = pd.concat([targets, targets_one_location])\n",
    "\n",
    "    features.reset_index(inplace=True, drop=True)\n",
    "    targets.reset_index(inplace=True, drop=True)\n",
    "\n",
    "    return features, targets['target_rides_next_hour']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 257/257 [00:02<00:00, 99.49it/s] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features.shape=(6168, 170)\n",
      "targets.shape=(6168,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "features, targets = transform_ts_data_into_features_and_target(\n",
    "    ts_data,\n",
    "    input_seq_len=24*7*1, # one week of history\n",
    "    step_size=24,\n",
    ")\n",
    "\n",
    "print(f'{features.shape=}')\n",
    "print(f'{targets.shape=}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b98d97558a062384a76b0309256306c9ce5dd4e2074fe66c33532239207fc923"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}