models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "e09d6ea1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import time\n",
    "from joblib import dump, load\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "from sklearn.metrics import roc_curve, roc_auc_score, precision_recall_curve, average_precision_score\n",
    "from sklearn.inspection import permutation_importance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f32ef63b",
   "metadata": {},
   "source": [
    "### Predicting gene essentiality from transposon sequencing data\n",
    "\n",
    "As a first pass, I will use the TraDIS transposon sequencing data as the ground truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "00a12e73",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('tnseq_features_essentiality.csv')\n",
    "data = data.set_index('Gene', drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e11fe652",
   "metadata": {},
   "outputs": [],
   "source": [
    "tradis_essentiality = data[data['TraDIS'].notnull()]['TraDIS']\n",
    "keio_essentiality = data[data['Keio'].notnull()]['Keio']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d13c83cd",
   "metadata": {},
   "source": [
    "The TraDIS data flags genes as nonessential, essential or unclear. For the purposes of this analysis, I will consider genes called unclear as nonessential.\n",
    "\n",
    "Essential: 1, \n",
    "Nonessential/Unclear: 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "c2b694a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_dict = {'Nonessential': 0, 'Unclear': 0, 'Essential': 1}\n",
    "tradis_label = tradis_essentiality.replace(label_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d4035c8",
   "metadata": {},
   "source": [
    "Extracting the features only for the genes for which we have a mapping to the TraDIS dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "006899cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_cols = data.columns[3:-2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "225e1774",
   "metadata": {},
   "outputs": [],
   "source": [
    "tradis_data = data.loc[data['TraDIS'].notnull(), feature_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "8d3cb42c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Mean_counts_5p_10pct</th>\n",
       "      <th>Mean_counts_3p_25pct</th>\n",
       "      <th>Mean_counts_interior</th>\n",
       "      <th>Insertion_index</th>\n",
       "      <th>Fraction_zeros</th>\n",
       "      <th>Fraction_above_thresh</th>\n",
       "      <th>Median_counts</th>\n",
       "      <th>Upper25</th>\n",
       "      <th>Lower25</th>\n",
       "      <th>Zeros_interval</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gene</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>33.000000</td>\n",
       "      <td>74.200000</td>\n",
       "      <td>73.368421</td>\n",
       "      <td>0.045847</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>0.859649</td>\n",
       "      <td>46.0</td>\n",
       "      <td>105.00</td>\n",
       "      <td>20.00</td>\n",
       "      <td>0.017544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44.000000</td>\n",
       "      <td>34.800000</td>\n",
       "      <td>63.647059</td>\n",
       "      <td>0.105063</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>0.764706</td>\n",
       "      <td>28.0</td>\n",
       "      <td>58.00</td>\n",
       "      <td>6.00</td>\n",
       "      <td>0.058824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.500000</td>\n",
       "      <td>39.636364</td>\n",
       "      <td>27.950000</td>\n",
       "      <td>0.033437</td>\n",
       "      <td>0.350000</td>\n",
       "      <td>0.550000</td>\n",
       "      <td>8.0</td>\n",
       "      <td>32.50</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>148.000000</td>\n",
       "      <td>44.000000</td>\n",
       "      <td>34.777778</td>\n",
       "      <td>0.180758</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>41.0</td>\n",
       "      <td>47.00</td>\n",
       "      <td>2.00</td>\n",
       "      <td>0.111111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>80.600000</td>\n",
       "      <td>85.909091</td>\n",
       "      <td>90.812500</td>\n",
       "      <td>0.180041</td>\n",
       "      <td>0.062500</td>\n",
       "      <td>0.937500</td>\n",
       "      <td>55.0</td>\n",
       "      <td>76.50</td>\n",
       "      <td>18.50</td>\n",
       "      <td>0.062500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4012</th>\n",
       "      <td>14.333333</td>\n",
       "      <td>56.500000</td>\n",
       "      <td>70.461538</td>\n",
       "      <td>0.229180</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>72.0</td>\n",
       "      <td>92.00</td>\n",
       "      <td>39.00</td>\n",
       "      <td>0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4013</th>\n",
       "      <td>51.000000</td>\n",
       "      <td>105.100000</td>\n",
       "      <td>52.142857</td>\n",
       "      <td>0.116429</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.928571</td>\n",
       "      <td>41.5</td>\n",
       "      <td>55.75</td>\n",
       "      <td>21.00</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4014</th>\n",
       "      <td>24.250000</td>\n",
       "      <td>38.437500</td>\n",
       "      <td>113.300000</td>\n",
       "      <td>0.122407</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>95.0</td>\n",
       "      <td>164.50</td>\n",
       "      <td>33.75</td>\n",
       "      <td>0.025000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4015</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>0.005372</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.416667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4016</th>\n",
       "      <td>52.500000</td>\n",
       "      <td>25.285714</td>\n",
       "      <td>24.500000</td>\n",
       "      <td>0.054945</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.5</td>\n",
       "      <td>28.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3618 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Mean_counts_5p_10pct  Mean_counts_3p_25pct  Mean_counts_interior  \\\n",
       "Gene                                                                     \n",
       "0                33.000000             74.200000             73.368421   \n",
       "1                44.000000             34.800000             63.647059   \n",
       "2                 7.500000             39.636364             27.950000   \n",
       "3               148.000000             44.000000             34.777778   \n",
       "4                80.600000             85.909091             90.812500   \n",
       "...                    ...                   ...                   ...   \n",
       "4012             14.333333             56.500000             70.461538   \n",
       "4013             51.000000            105.100000             52.142857   \n",
       "4014             24.250000             38.437500            113.300000   \n",
       "4015              0.000000              0.000000              2.500000   \n",
       "4016             52.500000             25.285714             24.500000   \n",
       "\n",
       "      Insertion_index  Fraction_zeros  Fraction_above_thresh  Median_counts  \\\n",
       "Gene                                                                          \n",
       "0            0.045847        0.105263               0.859649           46.0   \n",
       "1            0.105063        0.058824               0.764706           28.0   \n",
       "2            0.033437        0.350000               0.550000            8.0   \n",
       "3            0.180758        0.111111               0.666667           41.0   \n",
       "4            0.180041        0.062500               0.937500           55.0   \n",
       "...               ...             ...                    ...            ...   \n",
       "4012         0.229180        0.076923               0.923077           72.0   \n",
       "4013         0.116429        0.000000               0.928571           41.5   \n",
       "4014         0.122407        0.100000               0.875000           95.0   \n",
       "4015         0.005372        0.833333               0.083333            0.0   \n",
       "4016         0.054945        0.500000               0.458333            0.5   \n",
       "\n",
       "      Upper25  Lower25  Zeros_interval  \n",
       "Gene                                    \n",
       "0      105.00    20.00        0.017544  \n",
       "1       58.00     6.00        0.058824  \n",
       "2       32.50     0.00        0.100000  \n",
       "3       47.00     2.00        0.111111  \n",
       "4       76.50    18.50        0.062500  \n",
       "...       ...      ...             ...  \n",
       "4012    92.00    39.00        0.076923  \n",
       "4013    55.75    21.00        0.000000  \n",
       "4014   164.50    33.75        0.025000  \n",
       "4015     0.00     0.00        0.416667  \n",
       "4016    28.00     0.00        0.333333  \n",
       "\n",
       "[3618 rows x 10 columns]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tradis_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3948c3b",
   "metadata": {},
   "source": [
    "Great! This is the main dataframe that will be used for the machine learning models."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "152d53ab",
   "metadata": {},
   "source": [
    "### Defining functions for training and plotting results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "3ab0c959",
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_grid(X, y, model, grid_parameters, num_cv=5, randomState=42):\n",
    "    #we will pick the best model with 5-fold cross validation.\n",
    "#     model = RandomForestClassifier(class_weight={0:1, 1:2})\n",
    "    grid_search = GridSearchCV(model, grid_parameters, cv = num_cv, n_jobs=-1, verbose=True)\n",
    "    start = time.time()\n",
    "    grid_search.fit(X, y)    #fit the model with all the grid search parameters\n",
    "    end = time.time()\n",
    "    best_grid_clf = grid_search.best_estimator_   #pick the model with the best performance\n",
    "    #this is the score on the training/validation set\n",
    "    y_pred = best_grid_clf.predict(X)    \n",
    "    y_pred_score = best_grid_clf.predict_proba(X)[:,1]    #get the prediction score (so probability)\n",
    "    print(f\"Time taken for hyperparameter optimization is: {end-start}\")\n",
    "    print(\"The optimal model performance during the GridSearchCV is\")\n",
    "    print(confusion_matrix(y,y_pred))\n",
    "    print(classification_report(y,y_pred))\n",
    "    print(accuracy_score(y, y_pred))\n",
    "    \n",
    "    return best_grid_clf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "112d5046",
   "metadata": {},
   "source": [
    "Train/val (for convenience referred to as train) - test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "fbc8ed0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "testSize = 0.3    #keeping 30% of the genes\n",
    "X_train, X_test, y_train, y_test = train_test_split(tradis_data, tradis_label, test_size=testSize, random_state=42, stratify=tradis_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51c1e33d",
   "metadata": {},
   "source": [
    "Scaling the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "acc47133",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "X_train_scaled = pd.DataFrame(scaler.fit_transform(X_train), index = X_train.index, columns = X_train.columns)\n",
    "X_test_scaled = pd.DataFrame(scaler.transform(X_test), index = X_test.index, columns = X_test.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db68140b",
   "metadata": {},
   "source": [
    "I will try out a few different models for predicting gene essentiality:\n",
    "\n",
    "#### 1. Random forest classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "f47944d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_parameters_rf = {\n",
    "    'criterion': ['gini', 'entropy'],\n",
    "    'bootstrap': [True],\n",
    "    'oob_score': [True],\n",
    "    'max_depth': [3,5,7,9],\n",
    "    'max_samples': [0.25, 0.5, 0.75, 0.9],\n",
    "    'max_features': ['sqrt',  0.5, 0.75],\n",
    "    'n_estimators': [100, 150, 200, 400]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "2e8cb92d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 384 candidates, totalling 1920 fits\n",
      "Time taken for hyperparameter optimization is: 144.1589229106903\n",
      "The optimal model performance during the GridSearchCV is\n",
      "[[2265   34]\n",
      " [  33  200]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99      2299\n",
      "           1       0.85      0.86      0.86       233\n",
      "\n",
      "    accuracy                           0.97      2532\n",
      "   macro avg       0.92      0.92      0.92      2532\n",
      "weighted avg       0.97      0.97      0.97      2532\n",
      "\n",
      "0.9735387045813586\n"
     ]
    }
   ],
   "source": [
    "rf_model = RandomForestClassifier(class_weight='balanced')\n",
    "best_rf_model = optimize_grid(X_train_scaled, y_train, rf_model, grid_parameters_rf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "779df3c0",
   "metadata": {},
   "source": [
    "Now let's run the model on the test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "630ced34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model performance on unseen test set\n",
      "[[969  17]\n",
      " [ 15  85]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       986\n",
      "           1       0.83      0.85      0.84       100\n",
      "\n",
      "    accuracy                           0.97      1086\n",
      "   macro avg       0.91      0.92      0.91      1086\n",
      "weighted avg       0.97      0.97      0.97      1086\n",
      "\n",
      "0.9705340699815838\n"
     ]
    }
   ],
   "source": [
    "rf_y_pred = best_rf_model.predict(X_test_scaled)\n",
    "print(\"model performance on unseen test set\")\n",
    "print(confusion_matrix(y_test,rf_y_pred))\n",
    "print(classification_report(y_test,rf_y_pred))\n",
    "print(accuracy_score(y_test, rf_y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "b5c002b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.85 0.9827586206896551\n"
     ]
    }
   ],
   "source": [
    "tn, fp, fn, tp = confusion_matrix(y_test, rf_y_pred).ravel()\n",
    "specificity = tn / (tn+fp)\n",
    "sensitivity = tp / (tp+fn)\n",
    "print(sensitivity, specificity)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3726c26d",
   "metadata": {},
   "source": [
    "#### 2. Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "e51c9653",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
      "Time taken for hyperparameter optimization is: 0.5479259490966797\n",
      "The optimal model performance during the GridSearchCV is\n",
      "[[2148  151]\n",
      " [  13  220]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.93      0.96      2299\n",
      "           1       0.59      0.94      0.73       233\n",
      "\n",
      "    accuracy                           0.94      2532\n",
      "   macro avg       0.79      0.94      0.85      2532\n",
      "weighted avg       0.96      0.94      0.94      2532\n",
      "\n",
      "0.9352290679304898\n"
     ]
    }
   ],
   "source": [
    "grid_parameters_lr = {\n",
    "    'penalty': ['l2'],\n",
    "    'C': [0.1, 0.5, 1, 2, 10],\n",
    "    'solver': ['lbfgs','liblinear']\n",
    "}\n",
    "log_reg = LogisticRegression(class_weight='balanced')\n",
    "best_lr_model = optimize_grid(X_train_scaled, y_train, log_reg, grid_parameters_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "83eb9b33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model performance on unseen test set\n",
      "[[919  67]\n",
      " [  5  95]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.93      0.96       986\n",
      "           1       0.59      0.95      0.73       100\n",
      "\n",
      "    accuracy                           0.93      1086\n",
      "   macro avg       0.79      0.94      0.84      1086\n",
      "weighted avg       0.96      0.93      0.94      1086\n",
      "\n",
      "0.9337016574585635\n"
     ]
    }
   ],
   "source": [
    "lr_y_pred = best_lr_model.predict(X_test_scaled)\n",
    "print(\"model performance on unseen test set\")\n",
    "print(confusion_matrix(y_test,lr_y_pred))\n",
    "print(classification_report(y_test,lr_y_pred))\n",
    "print(accuracy_score(y_test, lr_y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40a8c7df",
   "metadata": {},
   "source": [
    "This is super interesting: logistic regression can pick out nearly all the essential genes but performs also misclassifies many genes as essential when they're not. This is definitely not a good model choice."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24cde18d",
   "metadata": {},
   "source": [
    "#### 3. Gradient boosted classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "0cd02d84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 288 candidates, totalling 1440 fits\n",
      "Time taken for hyperparameter optimization is: 122.91658616065979\n",
      "The optimal model performance during the GridSearchCV is\n",
      "[[2287   12]\n",
      " [   0  233]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      1.00      2299\n",
      "           1       0.95      1.00      0.97       233\n",
      "\n",
      "    accuracy                           1.00      2532\n",
      "   macro avg       0.98      1.00      0.99      2532\n",
      "weighted avg       1.00      1.00      1.00      2532\n",
      "\n",
      "0.995260663507109\n"
     ]
    }
   ],
   "source": [
    "grid_parameters_gb = {\n",
    "    'criterion': ['friedman_mse', 'squared_error'],\n",
    "    'learning_rate': [0.1, 0.2, 0.5],\n",
    "    'max_depth': [3,5,7,9],\n",
    "    'max_features': ['log2', 'sqrt', 0.25],\n",
    "    'n_estimators': [100, 150, 200, 400]\n",
    "}\n",
    "\n",
    "gb_model = GradientBoostingClassifier()\n",
    "best_gb_model = optimize_grid(X_train_scaled, y_train, gb_model, grid_parameters_gb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "0b2282e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model performance on unseen test set\n",
      "[[968  18]\n",
      " [ 15  85]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98       986\n",
      "           1       0.83      0.85      0.84       100\n",
      "\n",
      "    accuracy                           0.97      1086\n",
      "   macro avg       0.90      0.92      0.91      1086\n",
      "weighted avg       0.97      0.97      0.97      1086\n",
      "\n",
      "0.9696132596685083\n"
     ]
    }
   ],
   "source": [
    "gb_y_pred = best_gb_model.predict(X_test_scaled)\n",
    "print(\"model performance on unseen test set\")\n",
    "print(confusion_matrix(y_test,gb_y_pred))\n",
    "print(classification_report(y_test,gb_y_pred))\n",
    "print(accuracy_score(y_test, gb_y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "612d22f1",
   "metadata": {},
   "source": [
    "#### Let's compare this to a naive classification rule that I used in the LTEE TnSeq paper:\n",
    "\n",
    "if fraction_above_threshold < 0.1, call as essential:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "19bed617",
   "metadata": {},
   "outputs": [],
   "source": [
    "naive_pred = X_test['Fraction_above_thresh']<0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "d325cbee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model performance on validation set\n",
      "[[964  22]\n",
      " [ 13  87]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.98      0.98       986\n",
      "           1       0.80      0.87      0.83       100\n",
      "\n",
      "    accuracy                           0.97      1086\n",
      "   macro avg       0.89      0.92      0.91      1086\n",
      "weighted avg       0.97      0.97      0.97      1086\n",
      "\n",
      "0.9677716390423573\n"
     ]
    }
   ],
   "source": [
    "print(\"model performance on validation set\")\n",
    "print(confusion_matrix(y_test,naive_pred))\n",
    "print(classification_report(y_test,naive_pred))\n",
    "print(accuracy_score(y_test, naive_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7033b32",
   "metadata": {},
   "source": [
    "This is quite interesting: the machine learning models seem to do no better than a naive rule that I defined. \n",
    "\n",
    "Now, let's compare on the entire dataset (not that this is necessarily the fairest comparison)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "93b0e164",
   "metadata": {},
   "outputs": [],
   "source": [
    "naive_pred = tradis_data['Fraction_above_thresh']<0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "f057eb3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model performance on entire dataset\n",
      "[[3211   74]\n",
      " [  56  277]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98      3285\n",
      "           1       0.79      0.83      0.81       333\n",
      "\n",
      "    accuracy                           0.96      3618\n",
      "   macro avg       0.89      0.90      0.90      3618\n",
      "weighted avg       0.97      0.96      0.96      3618\n",
      "\n",
      "0.9640685461580984\n"
     ]
    }
   ],
   "source": [
    "print(\"model performance on entire dataset\")\n",
    "print(confusion_matrix(tradis_label,naive_pred))\n",
    "print(classification_report(tradis_label,naive_pred))\n",
    "print(accuracy_score(tradis_label, naive_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d78157a6",
   "metadata": {},
   "source": [
    "Looking at feature importance now:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "92140687",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = permutation_importance(best_rf_model, X_test_scaled, y_test, n_repeats=10, random_state=0, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "a5479ab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "imp = pd.DataFrame(result['importances_mean'], index=feature_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "651aa52e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Mean_counts_5p_10pct</th>\n",
       "      <td>-0.000552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_counts_3p_25pct</th>\n",
       "      <td>-0.002670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_counts_interior</th>\n",
       "      <td>0.004052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Insertion_index</th>\n",
       "      <td>0.014917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fraction_zeros</th>\n",
       "      <td>0.026519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fraction_above_thresh</th>\n",
       "      <td>-0.004328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Median_counts</th>\n",
       "      <td>0.000552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Upper25</th>\n",
       "      <td>0.002026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lower25</th>\n",
       "      <td>0.000368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Zeros_interval</th>\n",
       "      <td>-0.000921</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              0\n",
       "Mean_counts_5p_10pct  -0.000552\n",
       "Mean_counts_3p_25pct  -0.002670\n",
       "Mean_counts_interior   0.004052\n",
       "Insertion_index        0.014917\n",
       "Fraction_zeros         0.026519\n",
       "Fraction_above_thresh -0.004328\n",
       "Median_counts          0.000552\n",
       "Upper25                0.002026\n",
       "Lower25                0.000368\n",
       "Zeros_interval        -0.000921"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0dd98bf7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}