RF_Gbm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score \n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import classification_report \n",
    "import numpy as np\n",
    "from hyperopt import fmin, tpe, hp, STATUS_OK, Trials\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import f1_score\n",
    "from hyperopt import hp \n",
    "from hyperopt import fmin, tpe, Trials \n",
    "from sklearn.ensemble import GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_train = pd.read_csv('PCA_train_data.csv')\n",
    "pca_test = pd.read_csv('PCA_test_data.csv')\n",
    "tree_train = pd.read_csv('Tree_train_data.csv')\n",
    "tree_test= pd.read_csv('Tree_test_data.csv')\n",
    "trainset= pd.read_csv(\"train_dataset.csv\")\n",
    "testset= pd.read_csv('test_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>land</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>is_host_login</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>attack</th>\n",
       "      <th>protocol_type_icmp</th>\n",
       "      <th>protocol_type_tcp</th>\n",
       "      <th>protocol_type_udp</th>\n",
       "      <th>...</th>\n",
       "      <th>diff_srv_rate_scaled</th>\n",
       "      <th>srv_diff_host_rate_scaled</th>\n",
       "      <th>dst_host_count_scaled</th>\n",
       "      <th>dst_host_srv_count_scaled</th>\n",
       "      <th>dst_host_same_srv_rate_scaled</th>\n",
       "      <th>dst_host_diff_srv_rate_scaled</th>\n",
       "      <th>dst_host_same_src_port_rate_scaled</th>\n",
       "      <th>dst_host_srv_diff_host_rate_scaled</th>\n",
       "      <th>serror_rate_combined</th>\n",
       "      <th>rerror_rate_combined</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>neptune</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.101961</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.09</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 48 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   land  logged_in  root_shell  su_attempted  is_host_login  is_guest_login  \\\n",
       "0     0          0           0             0              0               0   \n",
       "1     0          0           0             0              0               0   \n",
       "2     0          0           0             0              0               0   \n",
       "3     0          1           0             0              0               0   \n",
       "4     0          1           0             0              0               0   \n",
       "\n",
       "    attack  protocol_type_icmp  protocol_type_tcp  protocol_type_udp  ...  \\\n",
       "0   normal                 0.0                1.0                0.0  ...   \n",
       "1   normal                 0.0                0.0                1.0  ...   \n",
       "2  neptune                 0.0                1.0                0.0  ...   \n",
       "3   normal                 0.0                1.0                0.0  ...   \n",
       "4   normal                 0.0                1.0                0.0  ...   \n",
       "\n",
       "   diff_srv_rate_scaled  srv_diff_host_rate_scaled  dst_host_count_scaled  \\\n",
       "0                  0.00                       0.00               0.588235   \n",
       "1                  0.15                       0.00               1.000000   \n",
       "2                  0.07                       0.00               1.000000   \n",
       "3                  0.00                       0.00               0.117647   \n",
       "4                  0.00                       0.09               1.000000   \n",
       "\n",
       "   dst_host_srv_count_scaled  dst_host_same_srv_rate_scaled  \\\n",
       "0                   0.098039                           0.17   \n",
       "1                   0.003922                           0.00   \n",
       "2                   0.101961                           0.10   \n",
       "3                   1.000000                           1.00   \n",
       "4                   1.000000                           1.00   \n",
       "\n",
       "   dst_host_diff_srv_rate_scaled  dst_host_same_src_port_rate_scaled  \\\n",
       "0                           0.03                                0.17   \n",
       "1                           0.60                                0.88   \n",
       "2                           0.05                                0.00   \n",
       "3                           0.00                                0.03   \n",
       "4                           0.00                                0.00   \n",
       "\n",
       "   dst_host_srv_diff_host_rate_scaled  serror_rate_combined  \\\n",
       "0                                0.00                  0.00   \n",
       "1                                0.00                  0.00   \n",
       "2                                0.00                  4.00   \n",
       "3                                0.04                  0.44   \n",
       "4                                0.00                  0.00   \n",
       "\n",
       "   rerror_rate_combined  \n",
       "0                  0.05  \n",
       "1                  0.00  \n",
       "2                  0.00  \n",
       "3                  0.01  \n",
       "4                  0.00  \n",
       "\n",
       "[5 rows x 48 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>land</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>is_host_login</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>attack</th>\n",
       "      <th>protocol_type_icmp</th>\n",
       "      <th>protocol_type_tcp</th>\n",
       "      <th>protocol_type_udp</th>\n",
       "      <th>...</th>\n",
       "      <th>is_attack</th>\n",
       "      <th>PCA1</th>\n",
       "      <th>PCA2</th>\n",
       "      <th>PCA3</th>\n",
       "      <th>PCA4</th>\n",
       "      <th>PCA5</th>\n",
       "      <th>PCA6</th>\n",
       "      <th>PCA7</th>\n",
       "      <th>PCA8</th>\n",
       "      <th>PCA9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.940599</td>\n",
       "      <td>-0.473843</td>\n",
       "      <td>0.719614</td>\n",
       "      <td>0.117456</td>\n",
       "      <td>-0.357645</td>\n",
       "      <td>-0.162415</td>\n",
       "      <td>-0.191790</td>\n",
       "      <td>-0.219253</td>\n",
       "      <td>-0.014388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.728731</td>\n",
       "      <td>-0.240017</td>\n",
       "      <td>1.426927</td>\n",
       "      <td>0.096360</td>\n",
       "      <td>0.159278</td>\n",
       "      <td>-0.026683</td>\n",
       "      <td>0.227241</td>\n",
       "      <td>0.402077</td>\n",
       "      <td>-0.355985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>neptune</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.035355</td>\n",
       "      <td>0.142109</td>\n",
       "      <td>-0.102676</td>\n",
       "      <td>-0.026129</td>\n",
       "      <td>-0.011964</td>\n",
       "      <td>0.012623</td>\n",
       "      <td>-0.021454</td>\n",
       "      <td>0.066945</td>\n",
       "      <td>-0.089175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.802288</td>\n",
       "      <td>-0.822607</td>\n",
       "      <td>-0.560944</td>\n",
       "      <td>0.151033</td>\n",
       "      <td>-0.207357</td>\n",
       "      <td>-0.232259</td>\n",
       "      <td>0.136882</td>\n",
       "      <td>0.050881</td>\n",
       "      <td>0.049239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.134026</td>\n",
       "      <td>-0.812825</td>\n",
       "      <td>-0.190227</td>\n",
       "      <td>-0.525232</td>\n",
       "      <td>0.102108</td>\n",
       "      <td>0.061269</td>\n",
       "      <td>-0.123393</td>\n",
       "      <td>0.092231</td>\n",
       "      <td>-0.040100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   land  logged_in  root_shell  su_attempted  is_host_login  is_guest_login  \\\n",
       "0     0          0           0             0              0               0   \n",
       "1     0          0           0             0              0               0   \n",
       "2     0          0           0             0              0               0   \n",
       "3     0          1           0             0              0               0   \n",
       "4     0          1           0             0              0               0   \n",
       "\n",
       "    attack  protocol_type_icmp  protocol_type_tcp  protocol_type_udp  ...  \\\n",
       "0   normal                 0.0                1.0                0.0  ...   \n",
       "1   normal                 0.0                0.0                1.0  ...   \n",
       "2  neptune                 0.0                1.0                0.0  ...   \n",
       "3   normal                 0.0                1.0                0.0  ...   \n",
       "4   normal                 0.0                1.0                0.0  ...   \n",
       "\n",
       "   is_attack      PCA1      PCA2      PCA3      PCA4      PCA5      PCA6  \\\n",
       "0        0.0 -0.940599 -0.473843  0.719614  0.117456 -0.357645 -0.162415   \n",
       "1        0.0 -0.728731 -0.240017  1.426927  0.096360  0.159278 -0.026683   \n",
       "2        1.0  3.035355  0.142109 -0.102676 -0.026129 -0.011964  0.012623   \n",
       "3        0.0 -0.802288 -0.822607 -0.560944  0.151033 -0.207357 -0.232259   \n",
       "4        0.0 -1.134026 -0.812825 -0.190227 -0.525232  0.102108  0.061269   \n",
       "\n",
       "       PCA7      PCA8      PCA9  \n",
       "0 -0.191790 -0.219253 -0.014388  \n",
       "1  0.227241  0.402077 -0.355985  \n",
       "2 -0.021454  0.066945 -0.089175  \n",
       "3  0.136882  0.050881  0.049239  \n",
       "4 -0.123393  0.092231 -0.040100  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca_train.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = trainset.drop(columns=['is_attack', 'attack_category',\"attack\"])\n",
    "y_train = trainset[\"attack_category\"]\n",
    "X_test = testset.drop(columns=['is_attack', 'attack_category',\"attack\"])\n",
    "y_test= testset[\"attack_category\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_pca_train = pca_train.drop(columns=['is_attack', 'attack_category',\"attack\"])\n",
    "y_pca_train = pca_train[\"attack_category\"]\n",
    "X_pca_test = pca_test.drop(columns=['is_attack', 'attack_category',\"attack\"])\n",
    "y_pca_test= pca_test[\"attack_category\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tree_train = tree_train.drop(columns=['is_attack', 'attack_category',\"attack\"])\n",
    "y_tree_train = tree_train[\"attack_category\"]\n",
    "X_tree_test = tree_test.drop(columns=['is_attack', 'attack_category',\"attack\"])\n",
    "y_tree_test= tree_test[\"attack_category\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>land</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>is_host_login</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>attack</th>\n",
       "      <th>protocol_type_icmp</th>\n",
       "      <th>protocol_type_tcp</th>\n",
       "      <th>protocol_type_udp</th>\n",
       "      <th>...</th>\n",
       "      <th>diff_srv_rate_scaled</th>\n",
       "      <th>srv_diff_host_rate_scaled</th>\n",
       "      <th>dst_host_count_scaled</th>\n",
       "      <th>dst_host_srv_count_scaled</th>\n",
       "      <th>dst_host_same_srv_rate_scaled</th>\n",
       "      <th>dst_host_diff_srv_rate_scaled</th>\n",
       "      <th>dst_host_same_src_port_rate_scaled</th>\n",
       "      <th>dst_host_srv_diff_host_rate_scaled</th>\n",
       "      <th>serror_rate_combined</th>\n",
       "      <th>rerror_rate_combined</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>neptune</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.101961</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.09</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125968</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>neptune</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125969</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.956863</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125970</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125971</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>neptune</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.031373</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125972</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.301961</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125973 rows × 48 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        land  logged_in  root_shell  su_attempted  is_host_login  \\\n",
       "0          0          0           0             0              0   \n",
       "1          0          0           0             0              0   \n",
       "2          0          0           0             0              0   \n",
       "3          0          1           0             0              0   \n",
       "4          0          1           0             0              0   \n",
       "...      ...        ...         ...           ...            ...   \n",
       "125968     0          0           0             0              0   \n",
       "125969     0          0           0             0              0   \n",
       "125970     0          1           0             0              0   \n",
       "125971     0          0           0             0              0   \n",
       "125972     0          1           0             0              0   \n",
       "\n",
       "        is_guest_login   attack  protocol_type_icmp  protocol_type_tcp  \\\n",
       "0                    0   normal                 0.0                1.0   \n",
       "1                    0   normal                 0.0                0.0   \n",
       "2                    0  neptune                 0.0                1.0   \n",
       "3                    0   normal                 0.0                1.0   \n",
       "4                    0   normal                 0.0                1.0   \n",
       "...                ...      ...                 ...                ...   \n",
       "125968               0  neptune                 0.0                1.0   \n",
       "125969               0   normal                 0.0                0.0   \n",
       "125970               0   normal                 0.0                1.0   \n",
       "125971               0  neptune                 0.0                1.0   \n",
       "125972               0   normal                 0.0                1.0   \n",
       "\n",
       "        protocol_type_udp  ...  diff_srv_rate_scaled  \\\n",
       "0                     0.0  ...                  0.00   \n",
       "1                     1.0  ...                  0.15   \n",
       "2                     0.0  ...                  0.07   \n",
       "3                     0.0  ...                  0.00   \n",
       "4                     0.0  ...                  0.00   \n",
       "...                   ...  ...                   ...   \n",
       "125968                0.0  ...                  0.06   \n",
       "125969                1.0  ...                  0.00   \n",
       "125970                0.0  ...                  0.00   \n",
       "125971                0.0  ...                  0.05   \n",
       "125972                0.0  ...                  0.00   \n",
       "\n",
       "        srv_diff_host_rate_scaled  dst_host_count_scaled  \\\n",
       "0                            0.00               0.588235   \n",
       "1                            0.00               1.000000   \n",
       "2                            0.00               1.000000   \n",
       "3                            0.00               0.117647   \n",
       "4                            0.09               1.000000   \n",
       "...                           ...                    ...   \n",
       "125968                       0.00               1.000000   \n",
       "125969                       0.00               1.000000   \n",
       "125970                       0.00               1.000000   \n",
       "125971                       0.00               1.000000   \n",
       "125972                       0.00               1.000000   \n",
       "\n",
       "        dst_host_srv_count_scaled  dst_host_same_srv_rate_scaled  \\\n",
       "0                        0.098039                           0.17   \n",
       "1                        0.003922                           0.00   \n",
       "2                        0.101961                           0.10   \n",
       "3                        1.000000                           1.00   \n",
       "4                        1.000000                           1.00   \n",
       "...                           ...                            ...   \n",
       "125968                   0.098039                           0.10   \n",
       "125969                   0.956863                           0.96   \n",
       "125970                   0.117647                           0.12   \n",
       "125971                   0.031373                           0.03   \n",
       "125972                   0.301961                           0.30   \n",
       "\n",
       "        dst_host_diff_srv_rate_scaled  dst_host_same_src_port_rate_scaled  \\\n",
       "0                                0.03                                0.17   \n",
       "1                                0.60                                0.88   \n",
       "2                                0.05                                0.00   \n",
       "3                                0.00                                0.03   \n",
       "4                                0.00                                0.00   \n",
       "...                               ...                                 ...   \n",
       "125968                           0.06                                0.00   \n",
       "125969                           0.01                                0.01   \n",
       "125970                           0.06                                0.00   \n",
       "125971                           0.05                                0.00   \n",
       "125972                           0.03                                0.30   \n",
       "\n",
       "        dst_host_srv_diff_host_rate_scaled  serror_rate_combined  \\\n",
       "0                                     0.00                  0.00   \n",
       "1                                     0.00                  0.00   \n",
       "2                                     0.00                  4.00   \n",
       "3                                     0.04                  0.44   \n",
       "4                                     0.00                  0.00   \n",
       "...                                    ...                   ...   \n",
       "125968                                0.00                  4.00   \n",
       "125969                                0.00                  0.00   \n",
       "125970                                0.00                  0.72   \n",
       "125971                                0.00                  4.00   \n",
       "125972                                0.00                  0.00   \n",
       "\n",
       "        rerror_rate_combined  \n",
       "0                       0.05  \n",
       "1                       0.00  \n",
       "2                       0.00  \n",
       "3                       0.01  \n",
       "4                       0.00  \n",
       "...                      ...  \n",
       "125968                  0.00  \n",
       "125969                  0.00  \n",
       "125970                  0.01  \n",
       "125971                  0.00  \n",
       "125972                  0.00  \n",
       "\n",
       "[125973 rows x 48 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>logged_in</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>protocol_type_icmp</th>\n",
       "      <th>protocol_type_tcp</th>\n",
       "      <th>protocol_type_udp</th>\n",
       "      <th>flag_REJ</th>\n",
       "      <th>flag_RSTO</th>\n",
       "      <th>flag_RSTOS0</th>\n",
       "      <th>flag_RSTR</th>\n",
       "      <th>...</th>\n",
       "      <th>diff_srv_rate_scaled</th>\n",
       "      <th>srv_diff_host_rate_scaled</th>\n",
       "      <th>dst_host_count_scaled</th>\n",
       "      <th>dst_host_srv_count_scaled</th>\n",
       "      <th>dst_host_same_srv_rate_scaled</th>\n",
       "      <th>dst_host_diff_srv_rate_scaled</th>\n",
       "      <th>dst_host_same_src_port_rate_scaled</th>\n",
       "      <th>dst_host_srv_diff_host_rate_scaled</th>\n",
       "      <th>serror_rate_combined</th>\n",
       "      <th>rerror_rate_combined</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.101961</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.09</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125968</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125969</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.956863</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125970</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125971</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.031373</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125972</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.301961</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125973 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        logged_in  root_shell  is_guest_login  protocol_type_icmp  \\\n",
       "0               0           0               0                 0.0   \n",
       "1               0           0               0                 0.0   \n",
       "2               0           0               0                 0.0   \n",
       "3               1           0               0                 0.0   \n",
       "4               1           0               0                 0.0   \n",
       "...           ...         ...             ...                 ...   \n",
       "125968          0           0               0                 0.0   \n",
       "125969          0           0               0                 0.0   \n",
       "125970          1           0               0                 0.0   \n",
       "125971          0           0               0                 0.0   \n",
       "125972          1           0               0                 0.0   \n",
       "\n",
       "        protocol_type_tcp  protocol_type_udp  flag_REJ  flag_RSTO  \\\n",
       "0                     1.0                0.0       0.0        0.0   \n",
       "1                     0.0                1.0       0.0        0.0   \n",
       "2                     1.0                0.0       0.0        0.0   \n",
       "3                     1.0                0.0       0.0        0.0   \n",
       "4                     1.0                0.0       0.0        0.0   \n",
       "...                   ...                ...       ...        ...   \n",
       "125968                1.0                0.0       0.0        0.0   \n",
       "125969                0.0                1.0       0.0        0.0   \n",
       "125970                1.0                0.0       0.0        0.0   \n",
       "125971                1.0                0.0       0.0        0.0   \n",
       "125972                1.0                0.0       0.0        0.0   \n",
       "\n",
       "        flag_RSTOS0  flag_RSTR  ...  diff_srv_rate_scaled  \\\n",
       "0               0.0        0.0  ...                  0.00   \n",
       "1               0.0        0.0  ...                  0.15   \n",
       "2               0.0        0.0  ...                  0.07   \n",
       "3               0.0        0.0  ...                  0.00   \n",
       "4               0.0        0.0  ...                  0.00   \n",
       "...             ...        ...  ...                   ...   \n",
       "125968          0.0        0.0  ...                  0.06   \n",
       "125969          0.0        0.0  ...                  0.00   \n",
       "125970          0.0        0.0  ...                  0.00   \n",
       "125971          0.0        0.0  ...                  0.05   \n",
       "125972          0.0        0.0  ...                  0.00   \n",
       "\n",
       "        srv_diff_host_rate_scaled  dst_host_count_scaled  \\\n",
       "0                            0.00               0.588235   \n",
       "1                            0.00               1.000000   \n",
       "2                            0.00               1.000000   \n",
       "3                            0.00               0.117647   \n",
       "4                            0.09               1.000000   \n",
       "...                           ...                    ...   \n",
       "125968                       0.00               1.000000   \n",
       "125969                       0.00               1.000000   \n",
       "125970                       0.00               1.000000   \n",
       "125971                       0.00               1.000000   \n",
       "125972                       0.00               1.000000   \n",
       "\n",
       "        dst_host_srv_count_scaled  dst_host_same_srv_rate_scaled  \\\n",
       "0                        0.098039                           0.17   \n",
       "1                        0.003922                           0.00   \n",
       "2                        0.101961                           0.10   \n",
       "3                        1.000000                           1.00   \n",
       "4                        1.000000                           1.00   \n",
       "...                           ...                            ...   \n",
       "125968                   0.098039                           0.10   \n",
       "125969                   0.956863                           0.96   \n",
       "125970                   0.117647                           0.12   \n",
       "125971                   0.031373                           0.03   \n",
       "125972                   0.301961                           0.30   \n",
       "\n",
       "        dst_host_diff_srv_rate_scaled  dst_host_same_src_port_rate_scaled  \\\n",
       "0                                0.03                                0.17   \n",
       "1                                0.60                                0.88   \n",
       "2                                0.05                                0.00   \n",
       "3                                0.00                                0.03   \n",
       "4                                0.00                                0.00   \n",
       "...                               ...                                 ...   \n",
       "125968                           0.06                                0.00   \n",
       "125969                           0.01                                0.01   \n",
       "125970                           0.06                                0.00   \n",
       "125971                           0.05                                0.00   \n",
       "125972                           0.03                                0.30   \n",
       "\n",
       "        dst_host_srv_diff_host_rate_scaled  serror_rate_combined  \\\n",
       "0                                     0.00                  0.00   \n",
       "1                                     0.00                  0.00   \n",
       "2                                     0.00                  4.00   \n",
       "3                                     0.04                  0.44   \n",
       "4                                     0.00                  0.00   \n",
       "...                                    ...                   ...   \n",
       "125968                                0.00                  4.00   \n",
       "125969                                0.00                  0.00   \n",
       "125970                                0.00                  0.72   \n",
       "125971                                0.00                  4.00   \n",
       "125972                                0.00                  0.00   \n",
       "\n",
       "        rerror_rate_combined  \n",
       "0                       0.05  \n",
       "1                       0.00  \n",
       "2                       0.00  \n",
       "3                       0.01  \n",
       "4                       0.00  \n",
       "...                      ...  \n",
       "125968                  0.00  \n",
       "125969                  0.00  \n",
       "125970                  0.01  \n",
       "125971                  0.00  \n",
       "125972                  0.00  \n",
       "\n",
       "[125973 rows x 36 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tree_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>land</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>is_host_login</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>protocol_type_icmp</th>\n",
       "      <th>protocol_type_tcp</th>\n",
       "      <th>protocol_type_udp</th>\n",
       "      <th>flag_OTH</th>\n",
       "      <th>...</th>\n",
       "      <th>flag_SH</th>\n",
       "      <th>PCA1</th>\n",
       "      <th>PCA2</th>\n",
       "      <th>PCA3</th>\n",
       "      <th>PCA4</th>\n",
       "      <th>PCA5</th>\n",
       "      <th>PCA6</th>\n",
       "      <th>PCA7</th>\n",
       "      <th>PCA8</th>\n",
       "      <th>PCA9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.940599</td>\n",
       "      <td>-0.473843</td>\n",
       "      <td>0.719614</td>\n",
       "      <td>0.117456</td>\n",
       "      <td>-0.357645</td>\n",
       "      <td>-0.162415</td>\n",
       "      <td>-0.191790</td>\n",
       "      <td>-0.219253</td>\n",
       "      <td>-0.014388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.728731</td>\n",
       "      <td>-0.240017</td>\n",
       "      <td>1.426927</td>\n",
       "      <td>0.096360</td>\n",
       "      <td>0.159278</td>\n",
       "      <td>-0.026683</td>\n",
       "      <td>0.227241</td>\n",
       "      <td>0.402077</td>\n",
       "      <td>-0.355985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.035355</td>\n",
       "      <td>0.142109</td>\n",
       "      <td>-0.102676</td>\n",
       "      <td>-0.026129</td>\n",
       "      <td>-0.011964</td>\n",
       "      <td>0.012623</td>\n",
       "      <td>-0.021454</td>\n",
       "      <td>0.066945</td>\n",
       "      <td>-0.089175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.802288</td>\n",
       "      <td>-0.822607</td>\n",
       "      <td>-0.560944</td>\n",
       "      <td>0.151033</td>\n",
       "      <td>-0.207357</td>\n",
       "      <td>-0.232259</td>\n",
       "      <td>0.136882</td>\n",
       "      <td>0.050881</td>\n",
       "      <td>0.049239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.134026</td>\n",
       "      <td>-0.812825</td>\n",
       "      <td>-0.190227</td>\n",
       "      <td>-0.525232</td>\n",
       "      <td>0.102108</td>\n",
       "      <td>0.061269</td>\n",
       "      <td>-0.123393</td>\n",
       "      <td>0.092231</td>\n",
       "      <td>-0.040100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125968</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.026111</td>\n",
       "      <td>0.134002</td>\n",
       "      <td>-0.102845</td>\n",
       "      <td>-0.034096</td>\n",
       "      <td>0.043781</td>\n",
       "      <td>-0.004933</td>\n",
       "      <td>-0.007050</td>\n",
       "      <td>-0.006698</td>\n",
       "      <td>0.012918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125969</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.119040</td>\n",
       "      <td>-0.792833</td>\n",
       "      <td>-0.115345</td>\n",
       "      <td>-0.513639</td>\n",
       "      <td>0.062364</td>\n",
       "      <td>-0.016443</td>\n",
       "      <td>-0.155788</td>\n",
       "      <td>0.095988</td>\n",
       "      <td>-0.082379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125970</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.211762</td>\n",
       "      <td>-0.391754</td>\n",
       "      <td>0.632769</td>\n",
       "      <td>-0.190737</td>\n",
       "      <td>-0.261550</td>\n",
       "      <td>-0.086665</td>\n",
       "      <td>-0.416727</td>\n",
       "      <td>-0.160747</td>\n",
       "      <td>0.057605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125971</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.059266</td>\n",
       "      <td>0.171533</td>\n",
       "      <td>0.003287</td>\n",
       "      <td>-0.001806</td>\n",
       "      <td>-0.018560</td>\n",
       "      <td>0.020528</td>\n",
       "      <td>-0.023829</td>\n",
       "      <td>-0.019866</td>\n",
       "      <td>-0.062215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125972</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.946390</td>\n",
       "      <td>-0.551146</td>\n",
       "      <td>0.698257</td>\n",
       "      <td>-0.175486</td>\n",
       "      <td>-0.031170</td>\n",
       "      <td>-0.088282</td>\n",
       "      <td>-0.303798</td>\n",
       "      <td>-0.084247</td>\n",
       "      <td>-0.112148</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125973 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        land  logged_in  root_shell  su_attempted  is_host_login  \\\n",
       "0          0          0           0             0              0   \n",
       "1          0          0           0             0              0   \n",
       "2          0          0           0             0              0   \n",
       "3          0          1           0             0              0   \n",
       "4          0          1           0             0              0   \n",
       "...      ...        ...         ...           ...            ...   \n",
       "125968     0          0           0             0              0   \n",
       "125969     0          0           0             0              0   \n",
       "125970     0          1           0             0              0   \n",
       "125971     0          0           0             0              0   \n",
       "125972     0          1           0             0              0   \n",
       "\n",
       "        is_guest_login  protocol_type_icmp  protocol_type_tcp  \\\n",
       "0                    0                 0.0                1.0   \n",
       "1                    0                 0.0                0.0   \n",
       "2                    0                 0.0                1.0   \n",
       "3                    0                 0.0                1.0   \n",
       "4                    0                 0.0                1.0   \n",
       "...                ...                 ...                ...   \n",
       "125968               0                 0.0                1.0   \n",
       "125969               0                 0.0                0.0   \n",
       "125970               0                 0.0                1.0   \n",
       "125971               0                 0.0                1.0   \n",
       "125972               0                 0.0                1.0   \n",
       "\n",
       "        protocol_type_udp  flag_OTH  ...  flag_SH      PCA1      PCA2  \\\n",
       "0                     0.0       0.0  ...      0.0 -0.940599 -0.473843   \n",
       "1                     1.0       0.0  ...      0.0 -0.728731 -0.240017   \n",
       "2                     0.0       0.0  ...      0.0  3.035355  0.142109   \n",
       "3                     0.0       0.0  ...      0.0 -0.802288 -0.822607   \n",
       "4                     0.0       0.0  ...      0.0 -1.134026 -0.812825   \n",
       "...                   ...       ...  ...      ...       ...       ...   \n",
       "125968                0.0       0.0  ...      0.0  3.026111  0.134002   \n",
       "125969                1.0       0.0  ...      0.0 -1.119040 -0.792833   \n",
       "125970                0.0       0.0  ...      0.0 -0.211762 -0.391754   \n",
       "125971                0.0       0.0  ...      0.0  3.059266  0.171533   \n",
       "125972                0.0       0.0  ...      0.0 -0.946390 -0.551146   \n",
       "\n",
       "            PCA3      PCA4      PCA5      PCA6      PCA7      PCA8      PCA9  \n",
       "0       0.719614  0.117456 -0.357645 -0.162415 -0.191790 -0.219253 -0.014388  \n",
       "1       1.426927  0.096360  0.159278 -0.026683  0.227241  0.402077 -0.355985  \n",
       "2      -0.102676 -0.026129 -0.011964  0.012623 -0.021454  0.066945 -0.089175  \n",
       "3      -0.560944  0.151033 -0.207357 -0.232259  0.136882  0.050881  0.049239  \n",
       "4      -0.190227 -0.525232  0.102108  0.061269 -0.123393  0.092231 -0.040100  \n",
       "...          ...       ...       ...       ...       ...       ...       ...  \n",
       "125968 -0.102845 -0.034096  0.043781 -0.004933 -0.007050 -0.006698  0.012918  \n",
       "125969 -0.115345 -0.513639  0.062364 -0.016443 -0.155788  0.095988 -0.082379  \n",
       "125970  0.632769 -0.190737 -0.261550 -0.086665 -0.416727 -0.160747  0.057605  \n",
       "125971  0.003287 -0.001806 -0.018560  0.020528 -0.023829 -0.019866 -0.062215  \n",
       "125972  0.698257 -0.175486 -0.031170 -0.088282 -0.303798 -0.084247 -0.112148  \n",
       "\n",
       "[125973 rows x 29 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_pca_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing Smote on PCA dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "from collections import Counter\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before SMOTE: Counter({0: 67343, 1: 45927, 2: 11656, 3: 995, 4: 52})\n",
      "After SMOTE: Counter({0: 67343, 1: 45927, 2: 11656, 3: 10000, 4: 5000})\n"
     ]
    }
   ],
   "source": [
    "print(\"Before SMOTE:\", Counter(y_pca_train))\n",
    "sampling_strategy = {3: 10000, 4: 5000}\n",
    "\n",
    "# Instantiating the SMOTE object\n",
    "smote = SMOTE(random_state = 42, sampling_strategy = sampling_strategy)\n",
    "\n",
    "# Applying SMOTE to generate synthetic samples\n",
    "X_pca_resampled, y_pca_resampled = smote.fit_resample(X_pca_train, y_pca_train)\n",
    "\n",
    "print(\"After SMOTE:\", Counter(y_pca_resampled))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, X, y):\n",
    "    skf = StratifiedKFold(n_splits=5)\n",
    "\n",
    "    accuracy_scores = []\n",
    "    precision_scores = []\n",
    "    recall_scores = []\n",
    "    f1_scores = []\n",
    "\n",
    "    for train_index, test_index in skf.split(X, y):\n",
    "        X_train, y_train = X.iloc[train_index], y.iloc[train_index]\n",
    "        X_test, y_test = X.iloc[test_index], y.iloc[test_index]\n",
    "\n",
    "        model.fit(X_train, y_train)\n",
    "\n",
    "        pred = model.predict(X_test)\n",
    "        accuracy = accuracy_score(y_test, pred)\n",
    "        precision = precision_score(y_test, pred, average='weighted')\n",
    "        recall = recall_score(y_test, pred, average='weighted') \n",
    "        f1 = f1_score(y_test, pred, average='weighted') \n",
    "\n",
    "        accuracy_scores.append(accuracy)\n",
    "        precision_scores.append(precision)\n",
    "        recall_scores.append(recall)\n",
    "        f1_scores.append(f1)\n",
    "\n",
    "    # print('{0} Accuracy: {1: .5f} Precision: {2: .5f} Recall: {3: .5f} F1_Score: {4: .5f}'.format(\n",
    "    #     model.__class__.__name__, \n",
    "    #     np.mean(accuracy_scores),\n",
    "    #     np.mean(precision_scores),\n",
    "    #     np.mean(recall_scores),\n",
    "    #     np.mean(f1_scores)))\n",
    "    \n",
    "    return -np.mean(f1_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, X_train, y_train, X_test, y_test):\n",
    "    \n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    y_pred = model.predict(X_test)\n",
    "    \n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    precision = precision_score(y_test, y_pred, average='weighted')\n",
    "    recall = recall_score(y_test, y_pred, average='weighted')\n",
    "    f1 = f1_score(y_test, y_pred, average='weighted')\n",
    "    \n",
    "    print(\"Model: \", model.__class__.__name__)\n",
    "    print(\"Accuracy: \", accuracy)\n",
    "    print(\"Precision: \", precision)\n",
    "    print(\"Recall: \", recall)\n",
    "    print(\"F1 Score: \", f1)\n",
    "    print(\"####################################\")\n",
    "    \n",
    "    # report = classification_report(y_test, y_pred)\n",
    "    # print(\"Classification Report:\\n\", report)\n",
    "    \n",
    "    return -f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def best_params(objective_func, search_space):\n",
    "    trials = Trials() \n",
    "\n",
    "    best = fmin(fn=objective_func,\n",
    "                space=search_space,\n",
    "                algo=tpe.suggest,\n",
    "                max_evals=20,\n",
    "                trials=trials,\n",
    "                rstate=np.random.default_rng(seed=30))\n",
    "\n",
    "    print('Best Parameters:', best)\n",
    "    return best "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  RandomForestClassifier\n",
      "Accuracy:  0.7654364797728885\n",
      "Precision:  0.8213316146576164\n",
      "Recall:  0.7654364797728885\n",
      "F1 Score:  0.7341330941956512\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7341330941956512"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf =RandomForestClassifier(n_estimators=100, random_state=42)\n",
    "evaluate(clf, X_pca_resampled,y_pca_resampled,X_pca_test,y_pca_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_search_space = {\n",
    "    'n_estimators': hp.quniform('n_estimators', 50, 500, 50),\n",
    "    'max_depth': hp.quniform('max_depth', 1, 10, 1),\n",
    "    'min_samples_leaf': hp.choice('min_samples_leaf', [1, 2, 4, 6,8]),\n",
    "    'min_samples_split': hp.choice('min_samples_split', [2, 5, 10,15]),\n",
    "}\n",
    "\n",
    "def objective_func_rf(search_space):\n",
    "    # Create and train a Random Forest classifier with the specified hyperparameters\n",
    "    rf_clf = RandomForestClassifier(\n",
    "        n_estimators=int(search_space['n_estimators']),\n",
    "        max_depth=int(search_space['max_depth']),\n",
    "        min_samples_split=search_space['min_samples_split'],\n",
    "        min_samples_leaf=search_space['min_samples_leaf'],\n",
    "        random_state=42\n",
    "    )\n",
    "\n",
    " \n",
    "    return train_model(rf_clf,X_pca_resampled,y_pca_resampled)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [1:28:44<00:00, 106.49s/trial, best loss: -0.9751116241684874]\n",
      "Best Parameters: {'max_depth': 10.0, 'min_samples_leaf': 0, 'min_samples_split': 2, 'n_estimators': 300.0}\n"
     ]
    }
   ],
   "source": [
    "rf_best = best_params(objective_func_rf, rf_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  RandomForestClassifier\n",
      "Accuracy:  0.7638839602555003\n",
      "Precision:  0.8221904668952357\n",
      "Recall:  0.7638839602555003\n",
      "F1 Score:  0.7399908472235582\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7399908472235582"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = RandomForestClassifier(\n",
    "    n_estimators=300,\n",
    "    max_depth=10,\n",
    "    min_samples_split=10,\n",
    "    min_samples_leaf=1,\n",
    "    random_state=42 \n",
    ")\n",
    "evaluate(clf, X_pca_resampled,y_pca_resampled,X_pca_test,y_pca_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RANDOMFOREST Classifier\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  RandomForestClassifier\n",
      "Accuracy:  0.7621540099361249\n",
      "Precision:  0.8237763007549375\n",
      "Recall:  0.7621540099361249\n",
      "F1 Score:  0.7164674293999976\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7164674293999976"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Default Model\n",
    "clf =RandomForestClassifier(n_estimators=100, random_state=42)\n",
    "evaluate(clf, X_train,y_train,X_test,y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_search_space = {\n",
    "    'n_estimators': hp.quniform('n_estimators', 50, 500, 50),\n",
    "    'max_depth': hp.quniform('max_depth', 1, 10, 1),\n",
    "    'min_samples_leaf': hp.choice('min_samples_leaf', [1, 4, 6, 8,10]),\n",
    "    'min_samples_split': hp.choice('min_samples_split', [2, 15, 20,30]),\n",
    "}\n",
    "\n",
    "def objective_func_rf(search_space):\n",
    "    # Create and train a Random Forest classifier with the specified hyperparameters\n",
    "    rf_clf = RandomForestClassifier(\n",
    "        n_estimators=int(search_space['n_estimators']),\n",
    "        max_depth=int(search_space['max_depth']),\n",
    "        min_samples_split=search_space['min_samples_split'],\n",
    "        min_samples_leaf=search_space['min_samples_leaf'],\n",
    "        random_state=42\n",
    "    )\n",
    "\n",
    " \n",
    "    return train_model(rf_clf,X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [10:44<00:00, 32.23s/trial, best loss: -0.9849622700834164]\n",
      "Best Parameters: {'max_depth': 8.0, 'min_samples_leaf': 3, 'min_samples_split': 1, 'n_estimators': 150.0}\n"
     ]
    }
   ],
   "source": [
    "rf_best = best_params(objective_func_rf, rf_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  RandomForestClassifier\n",
      "Accuracy:  0.7559882895670689\n",
      "Precision:  0.8211970429149197\n",
      "Recall:  0.7559882895670689\n",
      "F1 Score:  0.7170903547161447\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7170903547161447"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = RandomForestClassifier(\n",
    "    n_estimators=500,\n",
    "    max_depth=20,\n",
    "    min_samples_split=15,\n",
    "    min_samples_leaf=8,\n",
    "    random_state=42 \n",
    ")\n",
    "evaluate(clf, X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  RandomForestClassifier\n",
      "Accuracy:  0.764194464158978\n",
      "Precision:  0.8209953081361437\n",
      "Recall:  0.764194464158978\n",
      "F1 Score:  0.7279026168987286\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7279026168987286"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf =RandomForestClassifier(n_estimators=100, random_state=42)\n",
    "evaluate(clf, X_pca_train,y_pca_train,X_pca_test,y_pca_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_search_space = {\n",
    "    'n_estimators': hp.quniform('n_estimators', 50, 500, 50),\n",
    "    'max_depth': hp.quniform('max_depth', 1, 10, 1),\n",
    "    'min_samples_leaf': hp.choice('min_samples_leaf', [1, 2, 4, 6,8]),\n",
    "    'min_samples_split': hp.choice('min_samples_split', [2, 5, 10,15]),\n",
    "}\n",
    "\n",
    "def objective_func_rf(search_space):\n",
    "    # Create and train a Random Forest classifier with the specified hyperparameters\n",
    "    rf_clf = RandomForestClassifier(\n",
    "        n_estimators=int(search_space['n_estimators']),\n",
    "        max_depth=int(search_space['max_depth']),\n",
    "        min_samples_split=search_space['min_samples_split'],\n",
    "        min_samples_leaf=search_space['min_samples_leaf'],\n",
    "        random_state=42\n",
    "    )\n",
    "\n",
    " \n",
    "    return train_model(rf_clf,X_pca_train,y_pca_train)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [26:35<00:00, 79.75s/trial, best loss: -0.9758201385254532]\n",
      "Best Parameters: {'max_depth': 8.0, 'min_samples_leaf': 3, 'min_samples_split': 1, 'n_estimators': 150.0}\n"
     ]
    }
   ],
   "source": [
    "rf_best = best_params(objective_func_rf, rf_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  RandomForestClassifier\n",
      "Accuracy:  0.7592707594038325\n",
      "Precision:  0.8203990298424062\n",
      "Recall:  0.7592707594038325\n",
      "F1 Score:  0.7234885789382247\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7234885789382247"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Hyperparameters after trial and error from HyperOpt Best Params\n",
    "clf = RandomForestClassifier(\n",
    "    n_estimators=400,\n",
    "    max_depth=20,\n",
    "    min_samples_split=10,\n",
    "    min_samples_leaf=4,\n",
    "    random_state=42\n",
    ")\n",
    "evaluate( clf, X_pca_train,y_pca_train,X_pca_test,y_pca_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For Tree Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  RandomForestClassifier\n",
      "Accuracy:  0.761444286728176\n",
      "Precision:  0.8193291240070046\n",
      "Recall:  0.761444286728176\n",
      "F1 Score:  0.7157725235435279\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7157725235435279"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf =RandomForestClassifier(n_estimators=100, random_state=42)\n",
    "evaluate( clf, X_tree_train,y_tree_train,X_tree_test,y_tree_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_search_space = {\n",
    "    'n_estimators': hp.quniform('n_estimators', 50, 500, 50),\n",
    "    'max_depth': hp.quniform('max_depth', 1, 10, 1),\n",
    "    'min_samples_leaf': hp.choice('min_samples_leaf', [1, 2, 4, 6,8]),\n",
    "    'min_samples_split': hp.choice('min_samples_split', [2, 5, 10,15]),\n",
    "}\n",
    "\n",
    "def objective_func_rf2(search_space):\n",
    "    # Create and train a Random Forest classifier with the specified hyperparameters\n",
    "    rf_clf = RandomForestClassifier(\n",
    "        n_estimators=int(search_space['n_estimators']),\n",
    "        max_depth=int(search_space['max_depth']),\n",
    "        min_samples_split=search_space['min_samples_split'],\n",
    "        min_samples_leaf=search_space['min_samples_leaf'],\n",
    "        random_state=42\n",
    "    )\n",
    "\n",
    " \n",
    "    return train_model(rf_clf,X_tree_train,y_tree_train)\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [11:29<00:00, 34.48s/trial, best loss: -0.989203199228635]\n",
      "Best Parameters: {'max_depth': 8.0, 'min_samples_leaf': 3, 'min_samples_split': 1, 'n_estimators': 150.0}\n"
     ]
    }
   ],
   "source": [
    "rf2_best = best_params(objective_func_rf2, rf_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  RandomForestClassifier\n",
      "Accuracy:  0.7555003548616039\n",
      "Precision:  0.8208712695442787\n",
      "Recall:  0.7555003548616039\n",
      "F1 Score:  0.7175159890795794\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7175159890795794"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Hyperparameters after trial and error from HyperOpt Best Params\n",
    "clf = RandomForestClassifier(\n",
    "    n_estimators=500,\n",
    "    max_depth=20,\n",
    "    min_samples_split=20,\n",
    "    min_samples_leaf=8,\n",
    "    random_state=42\n",
    ")\n",
    "evaluate( clf, X_tree_train,y_tree_train,X_tree_test,y_tree_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GBM Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>land</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>is_host_login</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>protocol_type_icmp</th>\n",
       "      <th>protocol_type_tcp</th>\n",
       "      <th>protocol_type_udp</th>\n",
       "      <th>flag_OTH</th>\n",
       "      <th>...</th>\n",
       "      <th>diff_srv_rate_scaled</th>\n",
       "      <th>srv_diff_host_rate_scaled</th>\n",
       "      <th>dst_host_count_scaled</th>\n",
       "      <th>dst_host_srv_count_scaled</th>\n",
       "      <th>dst_host_same_srv_rate_scaled</th>\n",
       "      <th>dst_host_diff_srv_rate_scaled</th>\n",
       "      <th>dst_host_same_src_port_rate_scaled</th>\n",
       "      <th>dst_host_srv_diff_host_rate_scaled</th>\n",
       "      <th>serror_rate_combined</th>\n",
       "      <th>rerror_rate_combined</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.101961</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.09</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125968</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125969</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.956863</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125970</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125971</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.031373</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125972</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.301961</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125973 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        land  logged_in  root_shell  su_attempted  is_host_login  \\\n",
       "0          0          0           0             0              0   \n",
       "1          0          0           0             0              0   \n",
       "2          0          0           0             0              0   \n",
       "3          0          1           0             0              0   \n",
       "4          0          1           0             0              0   \n",
       "...      ...        ...         ...           ...            ...   \n",
       "125968     0          0           0             0              0   \n",
       "125969     0          0           0             0              0   \n",
       "125970     0          1           0             0              0   \n",
       "125971     0          0           0             0              0   \n",
       "125972     0          1           0             0              0   \n",
       "\n",
       "        is_guest_login  protocol_type_icmp  protocol_type_tcp  \\\n",
       "0                    0                 0.0                1.0   \n",
       "1                    0                 0.0                0.0   \n",
       "2                    0                 0.0                1.0   \n",
       "3                    0                 0.0                1.0   \n",
       "4                    0                 0.0                1.0   \n",
       "...                ...                 ...                ...   \n",
       "125968               0                 0.0                1.0   \n",
       "125969               0                 0.0                0.0   \n",
       "125970               0                 0.0                1.0   \n",
       "125971               0                 0.0                1.0   \n",
       "125972               0                 0.0                1.0   \n",
       "\n",
       "        protocol_type_udp  flag_OTH  ...  diff_srv_rate_scaled  \\\n",
       "0                     0.0       0.0  ...                  0.00   \n",
       "1                     1.0       0.0  ...                  0.15   \n",
       "2                     0.0       0.0  ...                  0.07   \n",
       "3                     0.0       0.0  ...                  0.00   \n",
       "4                     0.0       0.0  ...                  0.00   \n",
       "...                   ...       ...  ...                   ...   \n",
       "125968                0.0       0.0  ...                  0.06   \n",
       "125969                1.0       0.0  ...                  0.00   \n",
       "125970                0.0       0.0  ...                  0.00   \n",
       "125971                0.0       0.0  ...                  0.05   \n",
       "125972                0.0       0.0  ...                  0.00   \n",
       "\n",
       "        srv_diff_host_rate_scaled  dst_host_count_scaled  \\\n",
       "0                            0.00               0.588235   \n",
       "1                            0.00               1.000000   \n",
       "2                            0.00               1.000000   \n",
       "3                            0.00               0.117647   \n",
       "4                            0.09               1.000000   \n",
       "...                           ...                    ...   \n",
       "125968                       0.00               1.000000   \n",
       "125969                       0.00               1.000000   \n",
       "125970                       0.00               1.000000   \n",
       "125971                       0.00               1.000000   \n",
       "125972                       0.00               1.000000   \n",
       "\n",
       "        dst_host_srv_count_scaled  dst_host_same_srv_rate_scaled  \\\n",
       "0                        0.098039                           0.17   \n",
       "1                        0.003922                           0.00   \n",
       "2                        0.101961                           0.10   \n",
       "3                        1.000000                           1.00   \n",
       "4                        1.000000                           1.00   \n",
       "...                           ...                            ...   \n",
       "125968                   0.098039                           0.10   \n",
       "125969                   0.956863                           0.96   \n",
       "125970                   0.117647                           0.12   \n",
       "125971                   0.031373                           0.03   \n",
       "125972                   0.301961                           0.30   \n",
       "\n",
       "        dst_host_diff_srv_rate_scaled  dst_host_same_src_port_rate_scaled  \\\n",
       "0                                0.03                                0.17   \n",
       "1                                0.60                                0.88   \n",
       "2                                0.05                                0.00   \n",
       "3                                0.00                                0.03   \n",
       "4                                0.00                                0.00   \n",
       "...                               ...                                 ...   \n",
       "125968                           0.06                                0.00   \n",
       "125969                           0.01                                0.01   \n",
       "125970                           0.06                                0.00   \n",
       "125971                           0.05                                0.00   \n",
       "125972                           0.03                                0.30   \n",
       "\n",
       "        dst_host_srv_diff_host_rate_scaled  serror_rate_combined  \\\n",
       "0                                     0.00                  0.00   \n",
       "1                                     0.00                  0.00   \n",
       "2                                     0.00                  4.00   \n",
       "3                                     0.04                  0.44   \n",
       "4                                     0.00                  0.00   \n",
       "...                                    ...                   ...   \n",
       "125968                                0.00                  4.00   \n",
       "125969                                0.00                  0.00   \n",
       "125970                                0.00                  0.72   \n",
       "125971                                0.00                  4.00   \n",
       "125972                                0.00                  0.00   \n",
       "\n",
       "        rerror_rate_combined  \n",
       "0                       0.05  \n",
       "1                       0.00  \n",
       "2                       0.00  \n",
       "3                       0.01  \n",
       "4                       0.00  \n",
       "...                      ...  \n",
       "125968                  0.00  \n",
       "125969                  0.00  \n",
       "125970                  0.01  \n",
       "125971                  0.00  \n",
       "125972                  0.00  \n",
       "\n",
       "[125973 rows x 45 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>land</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>is_host_login</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>protocol_type_icmp</th>\n",
       "      <th>protocol_type_tcp</th>\n",
       "      <th>protocol_type_udp</th>\n",
       "      <th>flag_OTH</th>\n",
       "      <th>...</th>\n",
       "      <th>flag_SH</th>\n",
       "      <th>PCA1</th>\n",
       "      <th>PCA2</th>\n",
       "      <th>PCA3</th>\n",
       "      <th>PCA4</th>\n",
       "      <th>PCA5</th>\n",
       "      <th>PCA6</th>\n",
       "      <th>PCA7</th>\n",
       "      <th>PCA8</th>\n",
       "      <th>PCA9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.940599</td>\n",
       "      <td>-0.473843</td>\n",
       "      <td>0.719614</td>\n",
       "      <td>0.117456</td>\n",
       "      <td>-0.357645</td>\n",
       "      <td>-0.162415</td>\n",
       "      <td>-0.191790</td>\n",
       "      <td>-0.219253</td>\n",
       "      <td>-0.014388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.728731</td>\n",
       "      <td>-0.240017</td>\n",
       "      <td>1.426927</td>\n",
       "      <td>0.096360</td>\n",
       "      <td>0.159278</td>\n",
       "      <td>-0.026683</td>\n",
       "      <td>0.227241</td>\n",
       "      <td>0.402077</td>\n",
       "      <td>-0.355985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.035355</td>\n",
       "      <td>0.142109</td>\n",
       "      <td>-0.102676</td>\n",
       "      <td>-0.026129</td>\n",
       "      <td>-0.011964</td>\n",
       "      <td>0.012623</td>\n",
       "      <td>-0.021454</td>\n",
       "      <td>0.066945</td>\n",
       "      <td>-0.089175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.802288</td>\n",
       "      <td>-0.822607</td>\n",
       "      <td>-0.560944</td>\n",
       "      <td>0.151033</td>\n",
       "      <td>-0.207357</td>\n",
       "      <td>-0.232259</td>\n",
       "      <td>0.136882</td>\n",
       "      <td>0.050881</td>\n",
       "      <td>0.049239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.134026</td>\n",
       "      <td>-0.812825</td>\n",
       "      <td>-0.190227</td>\n",
       "      <td>-0.525232</td>\n",
       "      <td>0.102108</td>\n",
       "      <td>0.061269</td>\n",
       "      <td>-0.123393</td>\n",
       "      <td>0.092231</td>\n",
       "      <td>-0.040100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125968</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.026111</td>\n",
       "      <td>0.134002</td>\n",
       "      <td>-0.102845</td>\n",
       "      <td>-0.034096</td>\n",
       "      <td>0.043781</td>\n",
       "      <td>-0.004933</td>\n",
       "      <td>-0.007050</td>\n",
       "      <td>-0.006698</td>\n",
       "      <td>0.012918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125969</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.119040</td>\n",
       "      <td>-0.792833</td>\n",
       "      <td>-0.115345</td>\n",
       "      <td>-0.513639</td>\n",
       "      <td>0.062364</td>\n",
       "      <td>-0.016443</td>\n",
       "      <td>-0.155788</td>\n",
       "      <td>0.095988</td>\n",
       "      <td>-0.082379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125970</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.211762</td>\n",
       "      <td>-0.391754</td>\n",
       "      <td>0.632769</td>\n",
       "      <td>-0.190737</td>\n",
       "      <td>-0.261550</td>\n",
       "      <td>-0.086665</td>\n",
       "      <td>-0.416727</td>\n",
       "      <td>-0.160747</td>\n",
       "      <td>0.057605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125971</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.059266</td>\n",
       "      <td>0.171533</td>\n",
       "      <td>0.003287</td>\n",
       "      <td>-0.001806</td>\n",
       "      <td>-0.018560</td>\n",
       "      <td>0.020528</td>\n",
       "      <td>-0.023829</td>\n",
       "      <td>-0.019866</td>\n",
       "      <td>-0.062215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125972</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.946390</td>\n",
       "      <td>-0.551146</td>\n",
       "      <td>0.698257</td>\n",
       "      <td>-0.175486</td>\n",
       "      <td>-0.031170</td>\n",
       "      <td>-0.088282</td>\n",
       "      <td>-0.303798</td>\n",
       "      <td>-0.084247</td>\n",
       "      <td>-0.112148</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125973 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        land  logged_in  root_shell  su_attempted  is_host_login  \\\n",
       "0          0          0           0             0              0   \n",
       "1          0          0           0             0              0   \n",
       "2          0          0           0             0              0   \n",
       "3          0          1           0             0              0   \n",
       "4          0          1           0             0              0   \n",
       "...      ...        ...         ...           ...            ...   \n",
       "125968     0          0           0             0              0   \n",
       "125969     0          0           0             0              0   \n",
       "125970     0          1           0             0              0   \n",
       "125971     0          0           0             0              0   \n",
       "125972     0          1           0             0              0   \n",
       "\n",
       "        is_guest_login  protocol_type_icmp  protocol_type_tcp  \\\n",
       "0                    0                 0.0                1.0   \n",
       "1                    0                 0.0                0.0   \n",
       "2                    0                 0.0                1.0   \n",
       "3                    0                 0.0                1.0   \n",
       "4                    0                 0.0                1.0   \n",
       "...                ...                 ...                ...   \n",
       "125968               0                 0.0                1.0   \n",
       "125969               0                 0.0                0.0   \n",
       "125970               0                 0.0                1.0   \n",
       "125971               0                 0.0                1.0   \n",
       "125972               0                 0.0                1.0   \n",
       "\n",
       "        protocol_type_udp  flag_OTH  ...  flag_SH      PCA1      PCA2  \\\n",
       "0                     0.0       0.0  ...      0.0 -0.940599 -0.473843   \n",
       "1                     1.0       0.0  ...      0.0 -0.728731 -0.240017   \n",
       "2                     0.0       0.0  ...      0.0  3.035355  0.142109   \n",
       "3                     0.0       0.0  ...      0.0 -0.802288 -0.822607   \n",
       "4                     0.0       0.0  ...      0.0 -1.134026 -0.812825   \n",
       "...                   ...       ...  ...      ...       ...       ...   \n",
       "125968                0.0       0.0  ...      0.0  3.026111  0.134002   \n",
       "125969                1.0       0.0  ...      0.0 -1.119040 -0.792833   \n",
       "125970                0.0       0.0  ...      0.0 -0.211762 -0.391754   \n",
       "125971                0.0       0.0  ...      0.0  3.059266  0.171533   \n",
       "125972                0.0       0.0  ...      0.0 -0.946390 -0.551146   \n",
       "\n",
       "            PCA3      PCA4      PCA5      PCA6      PCA7      PCA8      PCA9  \n",
       "0       0.719614  0.117456 -0.357645 -0.162415 -0.191790 -0.219253 -0.014388  \n",
       "1       1.426927  0.096360  0.159278 -0.026683  0.227241  0.402077 -0.355985  \n",
       "2      -0.102676 -0.026129 -0.011964  0.012623 -0.021454  0.066945 -0.089175  \n",
       "3      -0.560944  0.151033 -0.207357 -0.232259  0.136882  0.050881  0.049239  \n",
       "4      -0.190227 -0.525232  0.102108  0.061269 -0.123393  0.092231 -0.040100  \n",
       "...          ...       ...       ...       ...       ...       ...       ...  \n",
       "125968 -0.102845 -0.034096  0.043781 -0.004933 -0.007050 -0.006698  0.012918  \n",
       "125969 -0.115345 -0.513639  0.062364 -0.016443 -0.155788  0.095988 -0.082379  \n",
       "125970  0.632769 -0.190737 -0.261550 -0.086665 -0.416727 -0.160747  0.057605  \n",
       "125971  0.003287 -0.001806 -0.018560  0.020528 -0.023829 -0.019866 -0.062215  \n",
       "125972  0.698257 -0.175486 -0.031170 -0.088282 -0.303798 -0.084247 -0.112148  \n",
       "\n",
       "[125973 rows x 29 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_pca_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>logged_in</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>protocol_type_icmp</th>\n",
       "      <th>protocol_type_tcp</th>\n",
       "      <th>protocol_type_udp</th>\n",
       "      <th>flag_REJ</th>\n",
       "      <th>flag_RSTO</th>\n",
       "      <th>flag_RSTOS0</th>\n",
       "      <th>flag_RSTR</th>\n",
       "      <th>...</th>\n",
       "      <th>diff_srv_rate_scaled</th>\n",
       "      <th>srv_diff_host_rate_scaled</th>\n",
       "      <th>dst_host_count_scaled</th>\n",
       "      <th>dst_host_srv_count_scaled</th>\n",
       "      <th>dst_host_same_srv_rate_scaled</th>\n",
       "      <th>dst_host_diff_srv_rate_scaled</th>\n",
       "      <th>dst_host_same_src_port_rate_scaled</th>\n",
       "      <th>dst_host_srv_diff_host_rate_scaled</th>\n",
       "      <th>serror_rate_combined</th>\n",
       "      <th>rerror_rate_combined</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.101961</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.09</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125968</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125969</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.956863</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125970</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125971</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.031373</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125972</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.301961</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125973 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        logged_in  root_shell  is_guest_login  protocol_type_icmp  \\\n",
       "0               0           0               0                 0.0   \n",
       "1               0           0               0                 0.0   \n",
       "2               0           0               0                 0.0   \n",
       "3               1           0               0                 0.0   \n",
       "4               1           0               0                 0.0   \n",
       "...           ...         ...             ...                 ...   \n",
       "125968          0           0               0                 0.0   \n",
       "125969          0           0               0                 0.0   \n",
       "125970          1           0               0                 0.0   \n",
       "125971          0           0               0                 0.0   \n",
       "125972          1           0               0                 0.0   \n",
       "\n",
       "        protocol_type_tcp  protocol_type_udp  flag_REJ  flag_RSTO  \\\n",
       "0                     1.0                0.0       0.0        0.0   \n",
       "1                     0.0                1.0       0.0        0.0   \n",
       "2                     1.0                0.0       0.0        0.0   \n",
       "3                     1.0                0.0       0.0        0.0   \n",
       "4                     1.0                0.0       0.0        0.0   \n",
       "...                   ...                ...       ...        ...   \n",
       "125968                1.0                0.0       0.0        0.0   \n",
       "125969                0.0                1.0       0.0        0.0   \n",
       "125970                1.0                0.0       0.0        0.0   \n",
       "125971                1.0                0.0       0.0        0.0   \n",
       "125972                1.0                0.0       0.0        0.0   \n",
       "\n",
       "        flag_RSTOS0  flag_RSTR  ...  diff_srv_rate_scaled  \\\n",
       "0               0.0        0.0  ...                  0.00   \n",
       "1               0.0        0.0  ...                  0.15   \n",
       "2               0.0        0.0  ...                  0.07   \n",
       "3               0.0        0.0  ...                  0.00   \n",
       "4               0.0        0.0  ...                  0.00   \n",
       "...             ...        ...  ...                   ...   \n",
       "125968          0.0        0.0  ...                  0.06   \n",
       "125969          0.0        0.0  ...                  0.00   \n",
       "125970          0.0        0.0  ...                  0.00   \n",
       "125971          0.0        0.0  ...                  0.05   \n",
       "125972          0.0        0.0  ...                  0.00   \n",
       "\n",
       "        srv_diff_host_rate_scaled  dst_host_count_scaled  \\\n",
       "0                            0.00               0.588235   \n",
       "1                            0.00               1.000000   \n",
       "2                            0.00               1.000000   \n",
       "3                            0.00               0.117647   \n",
       "4                            0.09               1.000000   \n",
       "...                           ...                    ...   \n",
       "125968                       0.00               1.000000   \n",
       "125969                       0.00               1.000000   \n",
       "125970                       0.00               1.000000   \n",
       "125971                       0.00               1.000000   \n",
       "125972                       0.00               1.000000   \n",
       "\n",
       "        dst_host_srv_count_scaled  dst_host_same_srv_rate_scaled  \\\n",
       "0                        0.098039                           0.17   \n",
       "1                        0.003922                           0.00   \n",
       "2                        0.101961                           0.10   \n",
       "3                        1.000000                           1.00   \n",
       "4                        1.000000                           1.00   \n",
       "...                           ...                            ...   \n",
       "125968                   0.098039                           0.10   \n",
       "125969                   0.956863                           0.96   \n",
       "125970                   0.117647                           0.12   \n",
       "125971                   0.031373                           0.03   \n",
       "125972                   0.301961                           0.30   \n",
       "\n",
       "        dst_host_diff_srv_rate_scaled  dst_host_same_src_port_rate_scaled  \\\n",
       "0                                0.03                                0.17   \n",
       "1                                0.60                                0.88   \n",
       "2                                0.05                                0.00   \n",
       "3                                0.00                                0.03   \n",
       "4                                0.00                                0.00   \n",
       "...                               ...                                 ...   \n",
       "125968                           0.06                                0.00   \n",
       "125969                           0.01                                0.01   \n",
       "125970                           0.06                                0.00   \n",
       "125971                           0.05                                0.00   \n",
       "125972                           0.03                                0.30   \n",
       "\n",
       "        dst_host_srv_diff_host_rate_scaled  serror_rate_combined  \\\n",
       "0                                     0.00                  0.00   \n",
       "1                                     0.00                  0.00   \n",
       "2                                     0.00                  4.00   \n",
       "3                                     0.04                  0.44   \n",
       "4                                     0.00                  0.00   \n",
       "...                                    ...                   ...   \n",
       "125968                                0.00                  4.00   \n",
       "125969                                0.00                  0.00   \n",
       "125970                                0.00                  0.72   \n",
       "125971                                0.00                  4.00   \n",
       "125972                                0.00                  0.00   \n",
       "\n",
       "        rerror_rate_combined  \n",
       "0                       0.05  \n",
       "1                       0.00  \n",
       "2                       0.00  \n",
       "3                       0.01  \n",
       "4                       0.00  \n",
       "...                      ...  \n",
       "125968                  0.00  \n",
       "125969                  0.00  \n",
       "125970                  0.01  \n",
       "125971                  0.00  \n",
       "125972                  0.00  \n",
       "\n",
       "[125973 rows x 36 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tree_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  GradientBoostingClassifier\n",
      "Accuracy:  0.7678317955997161\n",
      "Precision:  0.8243788697571973\n",
      "Recall:  0.7678317955997161\n",
      "F1 Score:  0.729153072234855\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.729153072234855"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=100,  random_state=42)\n",
    "evaluate(gbm, X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the search space for hyperparameters\n",
    "gbm_search_space = {\n",
    "    'n_estimators': hp.choice('n_estimators', [100, 200, 300, 400]),\n",
    "    'max_depth': hp.choice('max_depth', [3, 5, 7, 9, 11]),\n",
    "    'learning_rate': hp.choice('learning_rate', [0.01, 0.05, 0.1, 0.2, 0.3]),\n",
    "    'subsample': hp.choice('subsample', [0.5, 0.7,0.9])\n",
    "    \n",
    "}\n",
    "\n",
    "# Define the objective function for optimizing F1 score\n",
    "def objective_func_gbm(params):\n",
    "    gbm = GradientBoostingClassifier(\n",
    "        n_estimators=int(params['n_estimators']),\n",
    "        max_depth=int(params['max_depth']),\n",
    "        learning_rate=params['learning_rate'],\n",
    "        subsample=params['subsample'],\n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "\n",
    "    return train_model(gbm,X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [6:39:29<00:00, 1198.47s/trial, best loss: -0.9976002091942894]  \n",
      "Best Parameters: {'learning_rate': 0.012374515507867188, 'max_depth': 3, 'min_samples_leaf': 4, 'min_samples_split': 3, 'n_estimators': 3, 'subsample': 0.6739557674630375}\n"
     ]
    }
   ],
   "source": [
    "gbm_best = best_params(objective_func_gbm, gbm_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  GradientBoostingClassifier\n",
      "Accuracy:  0.754347054648687\n",
      "Precision:  0.8063306689532979\n",
      "Recall:  0.754347054648687\n",
      "F1 Score:  0.7068861090278146\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7068861090278146"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=400, learning_rate= 0.012374515507867188, max_depth=9,subsample= 0.6739557674630375, random_state=42)\n",
    "evaluate(gbm, X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  GradientBoostingClassifier\n",
      "Accuracy:  0.7800301632363378\n",
      "Precision:  0.8271996274325536\n",
      "Recall:  0.7800301632363378\n",
      "F1 Score:  0.7563681226678713\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7563681226678713"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=100, random_state=42)\n",
    "evaluate(gbm, X_pca_train,y_pca_train,X_pca_test,y_pca_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the search space for hyperparameters\n",
    "gbm_search_space = {\n",
    "    'n_estimators': hp.choice('n_estimators', [100, 200, 300, 400]),\n",
    "    'max_depth': hp.choice('max_depth', [3, 5, 7, 9]),\n",
    "    'learning_rate': hp.choice('learning_rate', [0.01, 0.05, 0.1, 0.3]),\n",
    "    \n",
    "    \n",
    "}\n",
    "\n",
    "# Define the objective function for optimizing F1 score\n",
    "def objective_func_gbm(params):\n",
    "    gbm = GradientBoostingClassifier(\n",
    "        n_estimators=int(params['n_estimators']),\n",
    "        max_depth=int(params['max_depth']),\n",
    "        learning_rate=params['learning_rate'],\n",
    "        \n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "\n",
    "    return train_model(gbm,X_pca_train,y_pca_train)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbm_best = best_params(objective_func_gbm, gbm_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  GradientBoostingClassifier\n",
      "Accuracy:  0.7625088715400994\n",
      "Precision:  0.7876217474537873\n",
      "Recall:  0.7625088715400994\n",
      "F1 Score:  0.7323141874884166\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7323141874884166"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=300, learning_rate= 0.1, max_depth=7,random_state=42)\n",
    "evaluate(gbm, X_pca_train,y_pca_train,X_pca_test,y_pca_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  GradientBoostingClassifier\n",
      "Accuracy:  0.7601579134137686\n",
      "Precision:  0.7858369232362487\n",
      "Recall:  0.7601579134137686\n",
      "F1 Score:  0.7220533928633788\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7220533928633788"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=400, learning_rate= 0.01, max_depth=7,random_state=42)\n",
    "evaluate(gbm, X_pca_train,y_pca_train,X_pca_test,y_pca_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  GradientBoostingClassifier\n",
      "Accuracy:  0.7332771469127041\n",
      "Precision:  0.7396226853763646\n",
      "Recall:  0.7332771469127041\n",
      "F1 Score:  0.7094529903040842\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7094529903040842"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=400, learning_rate= 0.3, max_depth=7,random_state=42)\n",
    "evaluate(gbm, X_pca_train,y_pca_train,X_pca_test,y_pca_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tree Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  GradientBoostingClassifier\n",
      "Accuracy:  0.7488023420865862\n",
      "Precision:  0.8113205003485532\n",
      "Recall:  0.7488023420865862\n",
      "F1 Score:  0.7036452715792411\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7036452715792411"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=100, learning_rate=0.1, max_depth=3, random_state=42)\n",
    "evaluate(gbm, X_tree_train,y_tree_train,X_tree_test,y_tree_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the search space for hyperparameters\n",
    "gbm_search_space = {\n",
    "    'n_estimators': hp.choice('n_estimators', [100, 200, 300, 400]),\n",
    "    'max_depth': hp.choice('max_depth', [3, 5, 7, 9]),\n",
    "    'learning_rate': hp.choice('learning_rate', [0.01, 0.05, 0.1, 0.3]),\n",
    "    \n",
    "    \n",
    "}\n",
    "\n",
    "# Define the objective function for optimizing F1 score\n",
    "def objective_func_gbm(params):\n",
    "    gbm = GradientBoostingClassifier(\n",
    "        n_estimators=int(params['n_estimators']),\n",
    "        max_depth=int(params['max_depth']),\n",
    "        learning_rate=params['learning_rate'],\n",
    "        \n",
    "        random_state=42\n",
    "    )\n",
    "    \n",
    "\n",
    "    return train_model(gbm,X_tree_train,y_tree_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [4:52:47<00:00, 351.34s/trial, best loss: -0.9895762709813092]  \n",
      "Best Parameters: {'learning_rate': 0.03523010317024397, 'max_depth': 10.0, 'min_samples_leaf': 0.10046592387004868, 'min_samples_split': 0.10655581077489396, 'n_estimators': 350.0, 'subsample': 0.9865122012440967}\n"
     ]
    }
   ],
   "source": [
    "gbm2_best = best_params(objective_func_gbm2, gbm_search_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  GradientBoostingClassifier\n",
      "Accuracy:  0.7341199432221434\n",
      "Precision:  0.796853127179646\n",
      "Recall:  0.7341199432221434\n",
      "F1 Score:  0.6930681907940133\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.6930681907940133"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=350, learning_rate= 0.03523010317024397, max_depth=10.0,min_samples_leaf=0.10046592387004868, min_samples_split= 0.10655581077489396, subsample=0.9865122012440967, random_state=0)\n",
    "evaluate(gbm, X_tree_train,y_tree_train,X_tree_test,y_tree_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  GradientBoostingClassifier\n",
      "Accuracy:  0.7687189496096523\n",
      "Precision:  0.8092477860316887\n",
      "Recall:  0.7687189496096523\n",
      "F1 Score:  0.7328192629980853\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7328192629980853"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=400, learning_rate= 0.3, max_depth=7,random_state=42)\n",
    "evaluate(gbm, X_tree_train,y_tree_train,X_tree_test,y_tree_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  GradientBoostingClassifier\n",
      "Accuracy:  0.7565205819730305\n",
      "Precision:  0.799188651182008\n",
      "Recall:  0.7565205819730305\n",
      "F1 Score:  0.7084699803290744\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7084699803290744"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=100, learning_rate= 0.01, max_depth=9,random_state=42)\n",
    "evaluate(gbm, X_tree_train,y_tree_train,X_tree_test,y_tree_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  GradientBoostingClassifier\n",
      "Accuracy:  0.7609119943222143\n",
      "Precision:  0.8092455602340636\n",
      "Recall:  0.7609119943222143\n",
      "F1 Score:  0.7160194927393677\n",
      "####################################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7160194927393677"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(n_estimators=300, learning_rate= 0.1, max_depth=7,random_state=42)\n",
    "evaluate(gbm, X_tree_train,y_tree_train,X_tree_test,y_tree_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}