04_transform_raw_data_into_features_and_targets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading file 2022-01\n",
      "Downloading file 2022-02\n",
      "Downloading file 2022-03\n",
      "Downloading file 2022-04\n",
      "Downloading file 2022-05\n",
      "Downloading file 2022-06\n",
      "Downloading file 2022-07\n",
      "Downloading file 2022-08\n",
      "Downloading file 2022-09\n",
      "Downloading file 2022-10\n",
      "Downloading file 2022-11\n",
      "Downloading file 2022-12\n",
      "2022-12 file is not available\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>pickup_location_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-01-01 00:35:40</td>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-01-01 00:33:43</td>\n",
       "      <td>236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-01-01 00:53:21</td>\n",
       "      <td>166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-01-01 00:25:21</td>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-01-01 00:36:48</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3252712</th>\n",
       "      <td>2022-11-30 23:17:09</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3252713</th>\n",
       "      <td>2022-11-30 23:48:48</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3252714</th>\n",
       "      <td>2022-11-30 23:04:36</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3252715</th>\n",
       "      <td>2022-11-30 23:18:37</td>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3252716</th>\n",
       "      <td>2022-11-30 23:30:50</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>36255131 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            pickup_datetime  pickup_location_id\n",
       "0       2022-01-01 00:35:40                 142\n",
       "1       2022-01-01 00:33:43                 236\n",
       "2       2022-01-01 00:53:21                 166\n",
       "3       2022-01-01 00:25:21                 114\n",
       "4       2022-01-01 00:36:48                  68\n",
       "...                     ...                 ...\n",
       "3252712 2022-11-30 23:17:09                 144\n",
       "3252713 2022-11-30 23:48:48                  45\n",
       "3252714 2022-11-30 23:04:36                 163\n",
       "3252715 2022-11-30 23:18:37                 161\n",
       "3252716 2022-11-30 23:30:50                  74\n",
       "\n",
       "[36255131 rows x 2 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from src.data import load_raw_data\n",
    "\n",
    "rides = load_raw_data(year=2022)\n",
    "rides"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 265/265 [00:01<00:00, 141.56it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pickup_hour</th>\n",
       "      <th>rides</th>\n",
       "      <th>pickup_location_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-01-01 00:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-01-01 01:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-01-01 02:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-01-01 03:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-01-01 04:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124235</th>\n",
       "      <td>2022-11-30 19:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124236</th>\n",
       "      <td>2022-11-30 20:00:00</td>\n",
       "      <td>5</td>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124237</th>\n",
       "      <td>2022-11-30 21:00:00</td>\n",
       "      <td>7</td>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124238</th>\n",
       "      <td>2022-11-30 22:00:00</td>\n",
       "      <td>5</td>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124239</th>\n",
       "      <td>2022-11-30 23:00:00</td>\n",
       "      <td>9</td>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2124240 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                pickup_hour  rides  pickup_location_id\n",
       "0       2022-01-01 00:00:00      0                   1\n",
       "1       2022-01-01 01:00:00      0                   1\n",
       "2       2022-01-01 02:00:00      0                   1\n",
       "3       2022-01-01 03:00:00      0                   1\n",
       "4       2022-01-01 04:00:00      1                   1\n",
       "...                     ...    ...                 ...\n",
       "2124235 2022-11-30 19:00:00      4                 265\n",
       "2124236 2022-11-30 20:00:00      5                 265\n",
       "2124237 2022-11-30 21:00:00      7                 265\n",
       "2124238 2022-11-30 22:00:00      5                 265\n",
       "2124239 2022-11-30 23:00:00      9                 265\n",
       "\n",
       "[2124240 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from src.data import transform_raw_data_into_ts_data\n",
    "\n",
    "ts_data = transform_raw_data_into_ts_data(rides)\n",
    "ts_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 265/265 [00:24<00:00, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features.shape=(81090, 674)\n",
      "targets.shape=(81090,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from src.data import transform_ts_data_into_features_and_target\n",
    "\n",
    "features, targets = transform_ts_data_into_features_and_target(\n",
    "    ts_data,\n",
    "    input_seq_len=24*28*1, # one month\n",
    "    step_size=24,\n",
    ")\n",
    "\n",
    "print(f'{features.shape=}')\n",
    "print(f'{targets.shape=}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabular_data = features\n",
    "tabular_data['target_rides_next_hour'] = targets\n",
    "\n",
    "from src.paths import TRANSFORMED_DATA_DIR\n",
    "tabular_data.to_parquet(TRANSFORMED_DATA_DIR / 'tabular_data.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b98d97558a062384a76b0309256306c9ce5dd4e2074fe66c33532239207fc923"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}