XGB_LogReg_LightGBM_Stacking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import StackingClassifier\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from collections import Counter\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from hyperopt import hp \n",
    "from hyperopt import fmin, tpe, Trials \n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "warnings.simplefilter(action='ignore', category=UserWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = pd.read_csv('Datasets/train_dataset.csv')\n",
    "test_dataset = pd.read_csv('Datasets/test_dataset.csv')\n",
    "\n",
    "PCA_train_dataset = pd.read_csv('Datasets/PCA_train_data.csv')\n",
    "PCA_test_dataset = pd.read_csv('Datasets/PCA_test_data.csv')\n",
    "\n",
    "Tree_train_dataset = pd.read_csv('Datasets/Tree_train_data.csv')\n",
    "Tree_test_dataset = pd.read_csv('Datasets/Tree_test_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = train_dataset.drop(['is_attack' , 'attack_category' , 'attack'], axis=1)\n",
    "y_train = train_dataset['attack_category']\n",
    "\n",
    "X_test = test_dataset.drop(['is_attack' , 'attack_category' , 'attack'], axis=1)\n",
    "y_test = test_dataset['attack_category']\n",
    "\n",
    "X_pca_train = PCA_train_dataset.drop(columns=['is_attack', 'attack_category',\"attack\"], axis = 1)\n",
    "y_pca_train = PCA_train_dataset[\"attack_category\"]\n",
    "\n",
    "X_pca_test = PCA_test_dataset.drop(columns=['is_attack', 'attack_category',\"attack\"], axis = 1)\n",
    "y_pca_test= PCA_test_dataset[\"attack_category\"]\n",
    "\n",
    "X_tree_train = Tree_train_dataset.drop(columns=['is_attack', 'attack_category',\"attack\"], axis = 1)\n",
    "y_tree_train = Tree_train_dataset[\"attack_category\"]\n",
    "\n",
    "X_tree_test = Tree_test_dataset.drop(columns=['is_attack', 'attack_category',\"attack\"], axis = 1)\n",
    "y_tree_test= Tree_test_dataset[\"attack_category\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, X, y):\n",
    "    skf = StratifiedKFold(n_splits=5)\n",
    "\n",
    "    accuracy_scores = []\n",
    "    precision_scores = []\n",
    "    recall_scores = []\n",
    "    f1_scores = []\n",
    "\n",
    "    for train_index, test_index in skf.split(X, y):\n",
    "        X_train, y_train = X.iloc[train_index], y.iloc[train_index]\n",
    "        X_test, y_test = X.iloc[test_index], y.iloc[test_index]\n",
    "\n",
    "        model.fit(X_train, y_train)\n",
    "\n",
    "        pred = model.predict(X_test)\n",
    "        accuracy = accuracy_score(y_test, pred)\n",
    "        precision = precision_score(y_test, pred, average='weighted')\n",
    "        recall = recall_score(y_test, pred, average='weighted') \n",
    "        f1 = f1_score(y_test, pred, average='weighted') \n",
    "\n",
    "        accuracy_scores.append(accuracy)\n",
    "        precision_scores.append(precision)\n",
    "        recall_scores.append(recall)\n",
    "        f1_scores.append(f1)\n",
    "\n",
    "    # print('{0} Accuracy: {1: .5f} Precision: {2: .5f} Recall: {3: .5f} F1_Score: {4: .5f}'.format(\n",
    "    #     model.__class__.__name__, \n",
    "    #     np.mean(accuracy_scores),\n",
    "    #     np.mean(precision_scores),\n",
    "    #     np.mean(recall_scores),\n",
    "    #     np.mean(f1_scores)))\n",
    "    \n",
    "    return -np.mean(f1_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, X_train, y_train, X_test, y_test):\n",
    "    \n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    y_pred = model.predict(X_test)\n",
    "    \n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    precision = precision_score(y_test, y_pred, average='weighted')\n",
    "    recall = recall_score(y_test, y_pred, average='weighted')\n",
    "    f1 = f1_score(y_test, y_pred, average='weighted')\n",
    "    \n",
    "    print(\"Model: \", model.__class__.__name__)\n",
    "    print(\"Accuracy: \", accuracy)\n",
    "    print(\"Precision: \", precision)\n",
    "    print(\"Recall: \", recall)\n",
    "    print(\"F1 Score: \", f1)\n",
    "    print(\"####################################\")\n",
    "    \n",
    "    # report = classification_report(y_test, y_pred)\n",
    "    # print(\"Classification Report:\\n\", report)\n",
    "    \n",
    "    return f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [],
   "source": [
    "def best_params(objective_func, search_space):\n",
    "    trials = Trials() \n",
    "\n",
    "    best = fmin(fn=objective_func,\n",
    "                space=search_space,\n",
    "                algo=tpe.suggest,\n",
    "                max_evals=20,\n",
    "                trials=trials,\n",
    "                rstate=np.random.default_rng(seed=30))\n",
    "\n",
    "    print('Best Parameters:', best)\n",
    "    return best "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter Tuning for XGB, LOGREG, LGBM"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Original Dataset - SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before SMOTE: Counter({0: 67343, 1: 45927, 2: 11656, 3: 995, 4: 52})\n",
      "After SMOTE: Counter({0: 67343, 1: 45927, 2: 11656, 3: 10000, 4: 5000})\n"
     ]
    }
   ],
   "source": [
    "print(\"Before SMOTE:\", Counter(y_train))\n",
    "sampling_strategy = {3: 10000, 4: 5000}\n",
    "\n",
    "# Instantiating the SMOTE object\n",
    "smote = SMOTE(random_state = 42, sampling_strategy = sampling_strategy)\n",
    "\n",
    "# Applying SMOTE to generate synthetic samples\n",
    "X_train_resampled, y_train_resampled = smote.fit_resample(X_train, y_train)\n",
    "\n",
    "print(\"After SMOTE:\", Counter(y_train_resampled))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost - Original Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [2:08:16<00:00, 384.84s/trial, best loss: -0.9991431830611784]   \n",
      "Best Parameters: {'colsample_bytree': 0.6947001151470121, 'learning_rate': 0.1903979884533492, 'max_depth': 4.0, 'min_child_weight': 3.0, 'n_estimators': 300.0, 'subsample': 0.8968355124649539}\n"
     ]
    }
   ],
   "source": [
    "# optimize f1_score\n",
    "xgboost_search_space = {\n",
    "    'learning_rate': hp.uniform('learning_rate', 0.01, 0.3),\n",
    "    'n_estimators': hp.quniform('n_estimators', 50, 500, 50),\n",
    "    'max_depth': hp.quniform('max_depth', 3, 10, 1),\n",
    "    'min_child_weight': hp.quniform('min_child_weight', 1, 10, 1),\n",
    "    'subsample': hp.uniform('subsample', 0.6, 1.0),\n",
    "    'colsample_bytree': hp.uniform('colsample_bytree', 0.6, 1.0),\n",
    "}\n",
    "\n",
    "def xgboost_objective_func(search_space):\n",
    "    xgboost_clf = xgb.XGBClassifier(\n",
    "        learning_rate=search_space['learning_rate'],\n",
    "        n_estimators=int(search_space['n_estimators']),\n",
    "        max_depth=int(search_space['max_depth']),\n",
    "        min_child_weight=int(search_space['min_child_weight']),\n",
    "        subsample=search_space['subsample'],\n",
    "        colsample_bytree=search_space['colsample_bytree'],\n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "    return train_model(xgboost_clf, X_train, y_train)\n",
    "\n",
    "xgboost_best = best_params(xgboost_objective_func, xgboost_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.6947001151470121,\n",
       " 'learning_rate': 0.1903979884533492,\n",
       " 'max_depth': 4.0,\n",
       " 'min_child_weight': 3.0,\n",
       " 'n_estimators': 300.0,\n",
       " 'subsample': 0.8968355124649539}"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgboost_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  XGBClassifier\n",
      "Accuracy:  0.771735273243435\n",
      "Precision:  0.8267240141848498\n",
      "Recall:  0.771735273243435\n",
      "F1 Score:  0.7307784698534782\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7307784698534782"
      ]
     },
     "execution_count": 304,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgboost_clf = xgb.XGBClassifier(\n",
    "    learning_rate=xgboost_best['learning_rate'],\n",
    "    n_estimators=int(xgboost_best['n_estimators']),\n",
    "    max_depth=int(xgboost_best['max_depth']),\n",
    "    min_child_weight=int(xgboost_best['min_child_weight']),\n",
    "    subsample=xgboost_best['subsample'],\n",
    "    colsample_bytree=xgboost_best['colsample_bytree'],\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "evaluate(xgboost_clf, X_train, y_train, X_test, y_test)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression - Original Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [06:47<00:00, 20.38s/trial, best loss: -0.9603507555240267]\n",
      "Best Parameters: {'C': 0.7041774471868815, 'max_iter': 450.0, 'solver': 2}\n"
     ]
    }
   ],
   "source": [
    "logreg_search_space = {\n",
    "    'C': hp.loguniform('C', -5, 2),\n",
    "    'solver': hp.choice('solver', ['liblinear', 'lbfgs', 'saga']),\n",
    "    'max_iter': hp.quniform('max_iter', 50, 500, 50),\n",
    "}\n",
    "\n",
    "def logreg_objective_func(search_space):\n",
    "    logreg_clf = LogisticRegression(\n",
    "        C=search_space['C'],\n",
    "        solver=search_space['solver'],\n",
    "        max_iter=int(search_space['max_iter']),\n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "    return train_model(logreg_clf, X_train, y_train)\n",
    "\n",
    "logreg_best = best_params(logreg_objective_func, logreg_search_space)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 0.7041774471868815, 'max_iter': 450.0, 'solver': 2}"
      ]
     },
     "execution_count": 306,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  LogisticRegression\n",
      "Accuracy:  0.7347409510290986\n",
      "Precision:  0.7451520047618645\n",
      "Recall:  0.7347409510290986\n",
      "F1 Score:  0.6880444014186261\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6880444014186261"
      ]
     },
     "execution_count": 307,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg_clf = LogisticRegression(\n",
    "    C=logreg_best['C'],\n",
    "    solver='saga',\n",
    "    max_iter=int(logreg_best['max_iter']),\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "evaluate(logreg_clf, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LightGBM - Original Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [04:54<00:00, 14.75s/trial, best loss: -0.9987556299061202]\n",
      "Best Parameters: {'colsample_bytree': 0.7421428951838873, 'learning_rate': 0.030571346892657877, 'max_depth': 7.0, 'n_estimators': 200.0, 'num_leaves': 30.0, 'subsample': 0.7006367676212871}\n"
     ]
    }
   ],
   "source": [
    "lgb_search_space = {\n",
    "    'learning_rate': hp.uniform('learning_rate', 0.01, 0.3),\n",
    "    'n_estimators': hp.quniform('n_estimators', 50, 500, 50),\n",
    "    'max_depth': hp.quniform('max_depth', 3, 10, 1),\n",
    "    'num_leaves': hp.quniform('num_leaves', 20, 100, 10),\n",
    "    'subsample': hp.uniform('subsample', 0.6, 1.0),\n",
    "    'colsample_bytree': hp.uniform('colsample_bytree', 0.6, 1.0),\n",
    "}\n",
    "\n",
    "def lgb_objective_func(search_space):\n",
    "    lgb_clf = lgb.LGBMClassifier(\n",
    "        learning_rate=search_space['learning_rate'],\n",
    "        n_estimators=int(search_space['n_estimators']),\n",
    "        max_depth=int(search_space['max_depth']),\n",
    "        num_leaves=int(search_space['num_leaves']),\n",
    "        subsample=search_space['subsample'],\n",
    "        colsample_bytree=search_space['colsample_bytree'],\n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "    return train_model(lgb_clf, X_train, y_train)\n",
    "\n",
    "lgb_best = best_params(lgb_objective_func, lgb_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.7421428951838873,\n",
       " 'learning_rate': 0.030571346892657877,\n",
       " 'max_depth': 7.0,\n",
       " 'n_estimators': 200.0,\n",
       " 'num_leaves': 30.0,\n",
       " 'subsample': 0.7006367676212871}"
      ]
     },
     "execution_count": 309,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgb_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  LGBMClassifier\n",
      "Accuracy:  0.75\n",
      "Precision:  0.8186114231426869\n",
      "Recall:  0.75\n",
      "F1 Score:  0.7095278534996156\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7095278534996156"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgb_clf = lgb.LGBMClassifier(\n",
    "    learning_rate=lgb_best['learning_rate'],\n",
    "    n_estimators=int(lgb_best['n_estimators']),\n",
    "    max_depth=int(lgb_best['max_depth']),\n",
    "    num_leaves=int(lgb_best['num_leaves']),\n",
    "    subsample=lgb_best['subsample'],\n",
    "    colsample_bytree=lgb_best['colsample_bytree'],\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "evaluate(lgb_clf, X_train, y_train, X_test, y_test)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PCA Dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost - PCA Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [4:13:33<00:00, 760.66s/trial, best loss: -0.9951946942223394]   \n",
      "Best Parameters: {'colsample_bytree': 0.7943552681682089, 'learning_rate': 0.296360168110823, 'max_depth': 6.0, 'min_child_weight': 2.0, 'n_estimators': 450.0, 'subsample': 0.7184859038205977}\n"
     ]
    }
   ],
   "source": [
    "# optimize f1_score\n",
    "xgboost_search_space = {\n",
    "    'learning_rate': hp.uniform('learning_rate', 0.01, 0.3),\n",
    "    'n_estimators': hp.quniform('n_estimators', 50, 500, 50),\n",
    "    'max_depth': hp.quniform('max_depth', 3, 10, 1),\n",
    "    'min_child_weight': hp.quniform('min_child_weight', 1, 10, 1),\n",
    "    'subsample': hp.uniform('subsample', 0.6, 1.0),\n",
    "    'colsample_bytree': hp.uniform('colsample_bytree', 0.6, 1.0),\n",
    "}\n",
    "\n",
    "def xgboost_objective_func(search_space):\n",
    "    xgboost_clf = xgb.XGBClassifier(\n",
    "        learning_rate=search_space['learning_rate'],\n",
    "        n_estimators=int(search_space['n_estimators']),\n",
    "        max_depth=int(search_space['max_depth']),\n",
    "        min_child_weight=int(search_space['min_child_weight']),\n",
    "        subsample=search_space['subsample'],\n",
    "        colsample_bytree=search_space['colsample_bytree'],\n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "    return train_model(xgboost_clf, X_pca_train, y_pca_train)\n",
    "\n",
    "xgboost_pca_best = best_params(xgboost_objective_func, xgboost_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.7943552681682089,\n",
       " 'learning_rate': 0.296360168110823,\n",
       " 'max_depth': 6.0,\n",
       " 'min_child_weight': 2.0,\n",
       " 'n_estimators': 450.0,\n",
       " 'subsample': 0.7184859038205977}"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgboost_pca_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  XGBClassifier\n",
      "Accuracy:  0.7622427253371186\n",
      "Precision:  0.8160068744163382\n",
      "Recall:  0.7622427253371186\n",
      "F1 Score:  0.734746336798068\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.734746336798068"
      ]
     },
     "execution_count": 313,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgboost_clf = xgb.XGBClassifier(\n",
    "    learning_rate=xgboost_pca_best['learning_rate'],\n",
    "    n_estimators=int(xgboost_pca_best['n_estimators']),\n",
    "    max_depth=int(xgboost_pca_best['max_depth']),\n",
    "    min_child_weight=int(xgboost_pca_best['min_child_weight']),\n",
    "    subsample=xgboost_pca_best['subsample'],\n",
    "    colsample_bytree=xgboost_pca_best['colsample_bytree'],\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "evaluate(xgboost_clf, X_pca_train, y_pca_train, X_pca_test, y_pca_test)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression - PCA Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [04:49<00:00, 14.49s/trial, best loss: -0.9431113401879945]\n",
      "Best Parameters: {'C': 0.7041774471868815, 'max_iter': 450.0, 'solver': 2}\n"
     ]
    }
   ],
   "source": [
    "logreg_search_space = {\n",
    "    'C': hp.loguniform('C', -5, 2),\n",
    "    'solver': hp.choice('solver', ['liblinear', 'lbfgs', 'saga']),\n",
    "    'max_iter': hp.quniform('max_iter', 50, 500, 50),\n",
    "}\n",
    "\n",
    "def logreg_objective_func(search_space):\n",
    "    logreg_clf = LogisticRegression(\n",
    "        C=search_space['C'],\n",
    "        solver=search_space['solver'],\n",
    "        max_iter=int(search_space['max_iter']),\n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "    return train_model(logreg_clf, X_pca_train, y_pca_train)\n",
    "\n",
    "logreg_pca_best = best_params(logreg_objective_func, logreg_search_space)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 0.7041774471868815, 'max_iter': 450.0, 'solver': 2}"
      ]
     },
     "execution_count": 315,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{'C': 0.7041774471868815, 'max_iter': 450.0, 'solver': 2}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 0.7041774471868815, 'max_iter': 450.0, 'solver': 2}"
      ]
     },
     "execution_count": 316,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg_pca_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  LogisticRegression\n",
      "Accuracy:  0.7457860184528035\n",
      "Precision:  0.7958965417573522\n",
      "Recall:  0.7457860184528035\n",
      "F1 Score:  0.7028068065123463\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7028068065123463"
      ]
     },
     "execution_count": 317,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg_clf = LogisticRegression(\n",
    "    C=logreg_pca_best['C'],\n",
    "    solver='saga',\n",
    "    max_iter=int(logreg_pca_best['max_iter']),\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "evaluate(logreg_clf, X_pca_train, y_pca_train, X_pca_test, y_pca_test)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Light GBM - PCA Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [05:45<00:00, 17.25s/trial, best loss: -0.9938732916966033]\n",
      "Best Parameters: {'colsample_bytree': 0.7971403330338507, 'learning_rate': 0.031232104259997354, 'max_depth': 8.0, 'n_estimators': 200.0, 'num_leaves': 40.0, 'subsample': 0.8614591695193656}\n"
     ]
    }
   ],
   "source": [
    "lgb_search_space = {\n",
    "    'learning_rate': hp.uniform('learning_rate', 0.01, 0.3),\n",
    "    'n_estimators': hp.quniform('n_estimators', 50, 500, 50),\n",
    "    'max_depth': hp.quniform('max_depth', 3, 10, 1),\n",
    "    'num_leaves': hp.quniform('num_leaves', 20, 100, 10),\n",
    "    'subsample': hp.uniform('subsample', 0.6, 1.0),\n",
    "    'colsample_bytree': hp.uniform('colsample_bytree', 0.6, 1.0),\n",
    "}\n",
    "\n",
    "def lgb_objective_func(search_space):\n",
    "    lgb_clf = lgb.LGBMClassifier(\n",
    "        learning_rate=search_space['learning_rate'],\n",
    "        n_estimators=int(search_space['n_estimators']),\n",
    "        max_depth=int(search_space['max_depth']),\n",
    "        num_leaves=int(search_space['num_leaves']),\n",
    "        subsample=search_space['subsample'],\n",
    "        colsample_bytree=search_space['colsample_bytree'],\n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "    return train_model(lgb_clf, X_pca_train, y_pca_train)\n",
    "\n",
    "lgb_pca_best = best_params(lgb_objective_func, lgb_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.7971403330338507,\n",
       " 'learning_rate': 0.031232104259997354,\n",
       " 'max_depth': 8.0,\n",
       " 'n_estimators': 200.0,\n",
       " 'num_leaves': 40.0,\n",
       " 'subsample': 0.8614591695193656}"
      ]
     },
     "execution_count": 319,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgb_pca_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  LGBMClassifier\n",
      "Accuracy:  0.7660131298793471\n",
      "Precision:  0.8138079501332512\n",
      "Recall:  0.7660131298793471\n",
      "F1 Score:  0.736568341369667\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.736568341369667"
      ]
     },
     "execution_count": 320,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgb_clf = lgb.LGBMClassifier(\n",
    "    learning_rate=lgb_pca_best['learning_rate'],\n",
    "    n_estimators=int(lgb_pca_best['n_estimators']),\n",
    "    max_depth=int(lgb_pca_best['max_depth']),\n",
    "    num_leaves=int(lgb_pca_best['num_leaves']),\n",
    "    subsample=lgb_pca_best['subsample'],\n",
    "    colsample_bytree=lgb_pca_best['colsample_bytree'],\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "evaluate(lgb_clf, X_pca_train, y_pca_train, X_pca_test, y_pca_test)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tree Dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost - Tree Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [1:24:12<00:00, 252.64s/trial, best loss: -0.9991657848464588]\n",
      "Best Parameters: {'colsample_bytree': 0.6947001151470121, 'learning_rate': 0.1903979884533492, 'max_depth': 4.0, 'min_child_weight': 3.0, 'n_estimators': 300.0, 'subsample': 0.8968355124649539}\n"
     ]
    }
   ],
   "source": [
    "# optimize f1_score\n",
    "xgboost_search_space = {\n",
    "    'learning_rate': hp.uniform('learning_rate', 0.01, 0.3),\n",
    "    'n_estimators': hp.quniform('n_estimators', 50, 500, 50),\n",
    "    'max_depth': hp.quniform('max_depth', 3, 10, 1),\n",
    "    'min_child_weight': hp.quniform('min_child_weight', 1, 10, 1),\n",
    "    'subsample': hp.uniform('subsample', 0.6, 1.0),\n",
    "    'colsample_bytree': hp.uniform('colsample_bytree', 0.6, 1.0),\n",
    "}\n",
    "\n",
    "def xgboost_objective_func(search_space):\n",
    "    xgboost_clf = xgb.XGBClassifier(\n",
    "        learning_rate=search_space['learning_rate'],\n",
    "        n_estimators=int(search_space['n_estimators']),\n",
    "        max_depth=int(search_space['max_depth']),\n",
    "        min_child_weight=int(search_space['min_child_weight']),\n",
    "        subsample=search_space['subsample'],\n",
    "        colsample_bytree=search_space['colsample_bytree'],\n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "    return train_model(xgboost_clf, X_tree_train, y_tree_train)\n",
    "\n",
    "xgboost_tree_best = best_params(xgboost_objective_func, xgboost_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  XGBClassifier\n",
      "Accuracy:  0.7754613200851668\n",
      "Precision:  0.8284324772446929\n",
      "Recall:  0.7754613200851668\n",
      "F1 Score:  0.7343409168130508\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7343409168130508"
      ]
     },
     "execution_count": 322,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgboost_clf = xgb.XGBClassifier(\n",
    "    learning_rate=xgboost_tree_best['learning_rate'],\n",
    "    n_estimators=int(xgboost_tree_best['n_estimators']),\n",
    "    max_depth=int(xgboost_tree_best['max_depth']),\n",
    "    min_child_weight=int(xgboost_tree_best['min_child_weight']),\n",
    "    subsample=xgboost_tree_best['subsample'],\n",
    "    colsample_bytree=xgboost_tree_best['colsample_bytree'],\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "evaluate(xgboost_clf, X_tree_train, y_tree_train, X_tree_test, y_tree_test)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression - Tree Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [05:29<00:00, 16.50s/trial, best loss: -0.9603079146464533]\n",
      "Best Parameters: {'C': 0.7041774471868815, 'max_iter': 450.0, 'solver': 2}\n"
     ]
    }
   ],
   "source": [
    "logreg_search_space = {\n",
    "    'C': hp.loguniform('C', -5, 2),\n",
    "    'solver': hp.choice('solver', ['liblinear', 'lbfgs', 'saga']),\n",
    "    'max_iter': hp.quniform('max_iter', 50, 500, 50),\n",
    "}\n",
    "\n",
    "def logreg_objective_func(search_space):\n",
    "    logreg_clf = LogisticRegression(\n",
    "        C=search_space['C'],\n",
    "        solver=search_space['solver'],\n",
    "        max_iter=int(search_space['max_iter']),\n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "    return train_model(logreg_clf, X_tree_train, y_tree_train)\n",
    "\n",
    "logreg_tree_best = best_params(logreg_objective_func, logreg_search_space)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  LogisticRegression\n",
      "Accuracy:  0.7344748048261178\n",
      "Precision:  0.7449249065708662\n",
      "Recall:  0.7344748048261178\n",
      "F1 Score:  0.6878405507132683\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6878405507132683"
      ]
     },
     "execution_count": 324,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg_clf = LogisticRegression(\n",
    "    C=logreg_tree_best['C'],\n",
    "    solver='saga',\n",
    "    max_iter=int(logreg_tree_best['max_iter']),\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "evaluate(logreg_clf, X_tree_train, y_tree_train, X_tree_test, y_tree_test)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LGBM - Tree Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [04:51<00:00, 14.59s/trial, best loss: -0.9988211819717405]\n",
      "Best Parameters: {'colsample_bytree': 0.7421428951838873, 'learning_rate': 0.030571346892657877, 'max_depth': 7.0, 'n_estimators': 200.0, 'num_leaves': 30.0, 'subsample': 0.7006367676212871}\n"
     ]
    }
   ],
   "source": [
    "lgb_search_space = {\n",
    "    'learning_rate': hp.uniform('learning_rate', 0.01, 0.3),\n",
    "    'n_estimators': hp.quniform('n_estimators', 50, 500, 50),\n",
    "    'max_depth': hp.quniform('max_depth', 3, 10, 1),\n",
    "    'num_leaves': hp.quniform('num_leaves', 20, 100, 10),\n",
    "    'subsample': hp.uniform('subsample', 0.6, 1.0),\n",
    "    'colsample_bytree': hp.uniform('colsample_bytree', 0.6, 1.0),\n",
    "}\n",
    "\n",
    "def lgb_objective_func(search_space):\n",
    "    lgb_clf = lgb.LGBMClassifier(\n",
    "        learning_rate=search_space['learning_rate'],\n",
    "        n_estimators=int(search_space['n_estimators']),\n",
    "        max_depth=int(search_space['max_depth']),\n",
    "        num_leaves=int(search_space['num_leaves']),\n",
    "        subsample=search_space['subsample'],\n",
    "        colsample_bytree=search_space['colsample_bytree'],\n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "    return train_model(lgb_clf, X_tree_train, y_tree_train)\n",
    "\n",
    "lgb_tree_best = best_params(lgb_objective_func, lgb_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  LGBMClassifier\n",
      "Accuracy:  0.7493789921930447\n",
      "Precision:  0.8152826143202561\n",
      "Recall:  0.7493789921930447\n",
      "F1 Score:  0.708959288221762\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.708959288221762"
      ]
     },
     "execution_count": 326,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgb_clf = lgb.LGBMClassifier(\n",
    "    learning_rate=lgb_tree_best['learning_rate'],\n",
    "    n_estimators=int(lgb_tree_best['n_estimators']),\n",
    "    max_depth=int(lgb_tree_best['max_depth']),\n",
    "    num_leaves=int(lgb_tree_best['num_leaves']),\n",
    "    subsample=lgb_tree_best['subsample'],\n",
    "    colsample_bytree=lgb_tree_best['colsample_bytree'],\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "evaluate(lgb_clf, X_tree_train, y_tree_train, X_tree_test, y_tree_test)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacking Classifiers"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Original Dataset - XGBoost and RF Stacking Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.6947001151470121,\n",
       " 'learning_rate': 0.1903979884533492,\n",
       " 'max_depth': 4.0,\n",
       " 'min_child_weight': 3.0,\n",
       " 'n_estimators': 300.0,\n",
       " 'subsample': 0.8968355124649539}"
      ]
     },
     "execution_count": 327,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgboost_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  StackingClassifier\n",
      "Accuracy:  0.7736870120652946\n",
      "Precision:  0.8282265091222754\n",
      "Recall:  0.7736870120652946\n",
      "F1 Score:  0.7329682832638851\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7329682832638851"
      ]
     },
     "execution_count": 328,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_forest = RandomForestClassifier(random_state=42)\n",
    "xgboost_clf = xgb.XGBClassifier(\n",
    "    learning_rate=xgboost_best['learning_rate'],\n",
    "    n_estimators=int(xgboost_best['n_estimators']),\n",
    "    max_depth=int(xgboost_best['max_depth']),\n",
    "    min_child_weight=int(xgboost_best['min_child_weight']),\n",
    "    subsample=xgboost_best['subsample'],\n",
    "    colsample_bytree=xgboost_best['colsample_bytree'],\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "estimators = [('random_forest', random_forest), ('xgboost', xgboost_clf)]\n",
    "stacking_classifier = StackingClassifier(estimators=estimators, final_estimator=RandomForestClassifier(random_state=42))\n",
    "\n",
    "evaluate(stacking_classifier, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA Dataset - GBM and LightGBM Classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.7971403330338507,\n",
       " 'learning_rate': 0.031232104259997354,\n",
       " 'max_depth': 8.0,\n",
       " 'n_estimators': 200.0,\n",
       " 'num_leaves': 40.0,\n",
       " 'subsample': 0.8614591695193656}"
      ]
     },
     "execution_count": 329,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgb_pca_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  StackingClassifier\n",
      "Accuracy:  0.757319020581973\n",
      "Precision:  0.8069776104730947\n",
      "Recall:  0.757319020581973\n",
      "F1 Score:  0.7248820697900789\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7248820697900789"
      ]
     },
     "execution_count": 331,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=100, random_state=42)\n",
    "lgb_clf = lgb.LGBMClassifier(\n",
    "    learning_rate=lgb_pca_best['learning_rate'],\n",
    "    n_estimators=int(lgb_pca_best['n_estimators']),\n",
    "    max_depth=int(lgb_pca_best['max_depth']),\n",
    "    num_leaves=int(lgb_pca_best['num_leaves']),\n",
    "    subsample=lgb_pca_best['subsample'],\n",
    "    colsample_bytree=lgb_pca_best['colsample_bytree'],\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "estimators = [('gbm', gbm), ('lgbm', lgb_clf)]\n",
    "stacking_classifier_pca = StackingClassifier(estimators=estimators, final_estimator=GradientBoostingClassifier(random_state=42))\n",
    "\n",
    "# Train the stacking classifier\n",
    "\n",
    "evaluate(stacking_classifier_pca, X_pca_train, y_pca_train, X_pca_test, y_pca_test)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tree Dataset - GBM and XGBoost Classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.6947001151470121,\n",
       " 'learning_rate': 0.1903979884533492,\n",
       " 'max_depth': 4.0,\n",
       " 'min_child_weight': 3.0,\n",
       " 'n_estimators': 300.0,\n",
       " 'subsample': 0.8968355124649539}"
      ]
     },
     "execution_count": 332,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgboost_tree_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  StackingClassifier\n",
      "Accuracy:  0.7766589779985805\n",
      "Precision:  0.8245000917812263\n",
      "Recall:  0.7766589779985805\n",
      "F1 Score:  0.7367621788448833\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7367621788448833"
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=400, learning_rate= 0.3, max_depth=7,random_state=42)\n",
    "xgboost_clf = xgb.XGBClassifier(\n",
    "    learning_rate=xgboost_tree_best['learning_rate'],\n",
    "    n_estimators=int(xgboost_tree_best['n_estimators']),\n",
    "    max_depth=int(xgboost_tree_best['max_depth']),\n",
    "    min_child_weight=int(xgboost_tree_best['min_child_weight']),\n",
    "    subsample=xgboost_tree_best['subsample'],\n",
    "    colsample_bytree=xgboost_tree_best['colsample_bytree'],\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "estimators = [('gbm', gbm), ('xgboost', xgboost_clf)]\n",
    "stacking_classifier_tree = StackingClassifier(estimators=estimators, final_estimator=GradientBoostingClassifier(random_state=42))\n",
    "\n",
    "evaluate(stacking_classifier_tree, X_tree_train, y_tree_train, X_tree_test, y_tree_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}