3 - model_selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "from sklearn.metrics import mean_absolute_percentage_error\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load y_train, y_test, X_train, X_test\n",
    "\n",
    "y_train = pd.read_pickle('dataframes/y-train.pkl')\n",
    "y_val = pd.read_pickle('dataframes/y-val.pkl')\n",
    "X_train = pd.read_pickle('dataframes/x-train.pkl')\n",
    "X_val = pd.read_pickle('dataframes/x-val.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE:  50122494232.54211\n",
      "MAE:  172921.71471694484\n",
      "MPE:  0.3477339711282637\n"
     ]
    }
   ],
   "source": [
    "# Get MSE, MAE and MPE for model where we predict the mean of the training set\n",
    "mean = y_train.mean()\n",
    "y_pred = np.full(len(y_val), mean)\n",
    "\n",
    "print(\"MSE: \", mean_squared_error(y_val, y_pred))\n",
    "print(\"MAE: \", mean_absolute_error(y_val, y_pred))\n",
    "print(\"MPE: \", mean_absolute_percentage_error(y_val, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Calculate f_regression scores for each feature\n",
    "# from sklearn.feature_selection import f_regression\n",
    "\n",
    "# # Sort feature names by f_regression scores\n",
    "# feature_names = X_train.columns\n",
    "# scores = f_regression(X_train, y_train)[0]\n",
    "# feature_scores = pd.DataFrame({\"feature\": feature_names, \"score\": scores})\n",
    "# feature_scores.sort_values(by=\"score\", ascending=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE for linear regression model 1 => 8499885744.418515\n",
      "MAE for linear regression model 1 => 69103.22634310293\n",
      "MAPE for linear regression model 1 => 0.1345694935012263\n"
     ]
    }
   ],
   "source": [
    "# Train Linear Regression and evaluate on validation set\n",
    "from sklearn.linear_model import LinearRegression\n",
    "lin_reg = LinearRegression()\n",
    "lin_reg.fit(X_train, y_train)\n",
    "\n",
    "# Print MSE and MAE\n",
    "y_val_pred = lin_reg.predict(X_val)\n",
    "print(\"MSE for linear regression model 1 =>\", mean_squared_error(y_val, y_val_pred))\n",
    "print(\"MAE for linear regression model 1 =>\", mean_absolute_error(y_val, y_val_pred))\n",
    "print(\"MAPE for linear regression model 1 =>\", mean_absolute_percentage_error(y_val, y_val_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE for ridge regression alpha 1 => 8499870041.984717\n",
      "MAE for ridge regression alpha 1 => 69102.9129123165\n",
      "MSE for ridge regression alpha 10 => 8500228827.740491\n",
      "MAE for ridge regression alpha 10 => 69104.44644854168\n",
      "MSE for ridge regression alpha 100 => 8513022182.256242\n",
      "MAE for ridge regression alpha 100 => 69144.09874053\n",
      "MSE for ridge regression alpha 1000 => 8650154765.658485\n",
      "MAE for ridge regression alpha 1000 => 69434.42809272427\n",
      "MSE for ridge regression alpha 10000 => 9363121887.674316\n",
      "MAE for ridge regression alpha 10000 => 71598.28678661454\n",
      "MSE for ridge regression alpha 10000 => 10539176137.759523\n",
      "MAE for ridge regression alpha 10000 => 75718.30375667359\n"
     ]
    }
   ],
   "source": [
    "# Train Ridge Regression and evalue on validation set\n",
    "from sklearn.linear_model import Ridge\n",
    "\n",
    "rr = Ridge(alpha=1).fit(X_train, y_train) \n",
    "\n",
    "y_val_pred = rr.predict(X_val)\n",
    "print(\"MSE for ridge regression alpha 1 =>\", mean_squared_error(y_val, y_val_pred))\n",
    "print(\"MAE for ridge regression alpha 1 =>\", mean_absolute_error(y_val, y_val_pred))\n",
    "\n",
    "rr = Ridge(alpha=10).fit(X_train, y_train) \n",
    "\n",
    "y_val_pred = rr.predict(X_val)\n",
    "print(\"MSE for ridge regression alpha 10 =>\", mean_squared_error(y_val, y_val_pred))\n",
    "print(\"MAE for ridge regression alpha 10 =>\", mean_absolute_error(y_val, y_val_pred))\n",
    "\n",
    "rr = Ridge(alpha=100).fit(X_train, y_train) \n",
    "\n",
    "y_val_pred = rr.predict(X_val)\n",
    "print(\"MSE for ridge regression alpha 100 =>\", mean_squared_error(y_val, y_val_pred))\n",
    "print(\"MAE for ridge regression alpha 100 =>\", mean_absolute_error(y_val, y_val_pred))\n",
    "\n",
    "rr = Ridge(alpha=1000).fit(X_train, y_train) \n",
    "\n",
    "y_val_pred = rr.predict(X_val)\n",
    "print(\"MSE for ridge regression alpha 1000 =>\", mean_squared_error(y_val, y_val_pred))\n",
    "print(\"MAE for ridge regression alpha 1000 =>\", mean_absolute_error(y_val, y_val_pred))\n",
    "\n",
    "rr = Ridge(alpha=10000).fit(X_train, y_train) \n",
    "\n",
    "y_val_pred = rr.predict(X_val)\n",
    "print(\"MSE for ridge regression alpha 10000 =>\", mean_squared_error(y_val, y_val_pred))\n",
    "print(\"MAE for ridge regression alpha 10000 =>\", mean_absolute_error(y_val, y_val_pred))\n",
    "\n",
    "rr = Ridge(alpha=100000).fit(X_train, y_train) \n",
    "\n",
    "y_val_pred = rr.predict(X_val)\n",
    "print(\"MSE for ridge regression alpha 10000 =>\", mean_squared_error(y_val, y_val_pred))\n",
    "print(\"MAE for ridge regression alpha 10000 =>\", mean_absolute_error(y_val, y_val_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Train support vector regression and evaluate on validation set\n",
    "from sklearn.svm import SVR\n",
    "\n",
    "svr = SVR(kernel=\"linear\", C=1, epsilon=0.1)\n",
    "svr.fit(X_train, y_train)\n",
    "\n",
    "y_val_pred = svr.predict(X_val)\n",
    "print(\"MSE for SVR =>\", mean_squared_error(y_val, y_val_pred))\n",
    "print(\"MAE for SVR =>\", mean_absolute_error(y_val, y_val_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE for decision tree regression model 1 => 3512852739.4432545\n",
      "MAE for decision tree regression model 1 => 38416.6096755659\n",
      "MAPE for decision tree regression model 1 => 0.06905041035848657\n",
      "63\n"
     ]
    }
   ],
   "source": [
    "# Train Decison Tree Regressor and evaluate on validation set\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "tree_reg = DecisionTreeRegressor(random_state=42).fit(X_train, y_train)\n",
    "\n",
    "y_val_pred = tree_reg.predict(X_val)\n",
    "print(\"MSE for decision tree regression model 1 =>\", mean_squared_error(y_val, y_val_pred))\n",
    "print(\"MAE for decision tree regression model 1 =>\", mean_absolute_error(y_val, y_val_pred))\n",
    "print(\"MAPE for decision tree regression model 1 =>\", mean_absolute_percentage_error(y_val, y_val_pred))\n",
    "\n",
    "print(tree_reg.get_depth())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train using k-nearest neighbors\n",
    "# from sklearn.neighbors import KNeighborsRegressor\n",
    "\n",
    "# knn_reg = KNeighborsRegressor(weights=\"distance\", n_jobs=8).fit(X_train, y_train)\n",
    "\n",
    "# y_val_pred = knn_reg.predict(X_val)\n",
    "# print(\"MSE for k-nearst neighbors regression model 1 =>\", mean_squared_error(y_val, y_val_pred))\n",
    "# print(\"MAE for k-nearst neighbors regression model 1 =>\", mean_absolute_error(y_val, y_val_pred))\n",
    "# print(\"MAPE for k-nearst neighbors regression model 1 =>\", mean_absolute_percentage_error(y_val, y_val_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Train Linear support vector regressor and evaluate on validation set\n",
    "# from sklearn.svm import LinearSVR\n",
    "\n",
    "# svm_reg = LinearSVR(random_state=42).fit(X_train, y_train)\n",
    "\n",
    "# y_val_pred = svm_reg.predict(X_val)\n",
    "# print(\"MSE for linear support vector regression model 1 =>\", mean_squared_error(y_val, y_val_pred))\n",
    "# print(\"MAE for linear support vector regression model 1 =>\", mean_absolute_error(y_val, y_val_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE for random forest regression model 1 => 2092233181.4044025\n",
      "MAE for random forest regression model 1 => 30038.676611209292\n",
      "MAPE for random forest regression model 1 => 0.05468338536735954\n"
     ]
    }
   ],
   "source": [
    "# Train Random Forest Regressor and evaluate on validation set\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "forest_reg = RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=8).fit(X_train, y_train)\n",
    "\n",
    "y_val_pred = forest_reg.predict(X_val)\n",
    "print(\"MSE for random forest regression model 1 =>\", mean_squared_error(y_val, y_val_pred))\n",
    "print(\"MAE for random forest regression model 1 =>\", mean_absolute_error(y_val, y_val_pred))\n",
    "print(\"MAPE for random forest regression model 1 =>\", mean_absolute_percentage_error(y_val, y_val_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "max_depth = 0\n",
    "for dt_reg in forest_reg.estimators_:\n",
    "\tif dt_reg.get_depth() > max_depth:\n",
    "\t\tmax_depth = dt_reg.get_depth()\n",
    "\n",
    "print(max_depth)\n",
    "print(len(forest_reg.estimators_))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}