JITLine/JITLine_RQ1-RQ3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "from my_util import *\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import confusion_matrix, roc_auc_score, matthews_corrcoef, precision_recall_fscore_support, classification_report, auc\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "import numpy as np\n",
    "from scipy.optimize import differential_evolution\n",
    "import pandas as pd\n",
    "import time, pickle, math, warnings, os\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "\n",
    "\n",
    "projects = ['openstack','qt']\n",
    "sampling_methods = 'DE_SMOTE_min_df_3'\n",
    "\n",
    "remove_python_common_tokens = True\n",
    "\n",
    "create_path_if_not_exist('./data/')\n",
    "create_path_if_not_exist('./final_model/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_combined_df(code_commit, commit_id, label, metrics_df, count_vect):\n",
    "    code_df = pd.DataFrame()\n",
    "    code_df['commit_id'] = commit_id\n",
    "    code_df['code'] = code_commit\n",
    "    code_df['label'] = label\n",
    "    \n",
    "    code_df = code_df.sort_values(by='commit_id')\n",
    "    \n",
    "    metrics_df = metrics_df.sort_values(by='commit_id')\n",
    "    metrics_df = metrics_df.drop('commit_id',axis=1)\n",
    "    \n",
    "    code_change_arr = count_vect.transform(code_df['code']).astype(np.int16).toarray()\n",
    "    metrics_df_arr = metrics_df.to_numpy(dtype=np.float32)\n",
    "    \n",
    "    final_features = np.concatenate((code_change_arr,metrics_df_arr),axis=1)\n",
    "\n",
    "    return final_features, list(code_df['commit_id']), list(code_df['label'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective_func(k, train_feature, train_label, valid_feature, valid_label):\n",
    "    smote = SMOTE(random_state=42, k_neighbors= int(np.round(k)), n_jobs=32)\n",
    "    train_feature_res, train_label_res = smote.fit_resample(train_feature, train_label)\n",
    "    \n",
    "    clf = RandomForestClassifier(n_estimators=300, random_state=42, n_jobs=-1)\n",
    "    clf.fit(train_feature_res, train_label_res)\n",
    "    \n",
    "    prob = clf.predict_proba(valid_feature)[:,1]\n",
    "    auc = roc_auc_score(valid_label, prob)\n",
    "    \n",
    "    return -auc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The code below this cell is used to\n",
    "\n",
    "1. obtain the best k_neighbor of SMOTE (the value is rounded to int)\n",
    "2. resample train data using SMOTE with the best k_neighbor value\n",
    "3. train RF model and obtain prediction result from the model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_experiment(cur_proj):\n",
    "    data_path = './data/'\n",
    "    model_path = './final_model/'\n",
    "        \n",
    "    train_code, train_commit, train_label = prepare_data(cur_proj, mode='train',\n",
    "                                                                  remove_python_common_tokens=remove_python_common_tokens)\n",
    "    test_code, test_commit, test_label = prepare_data(cur_proj, mode='test',\n",
    "                                                              remove_python_common_tokens=remove_python_common_tokens)\n",
    "\n",
    "    commit_metrics = load_change_metrics_df(cur_proj)\n",
    "    train_commit_metrics = commit_metrics[commit_metrics['commit_id'].isin(train_commit)]\n",
    "    test_commit_metrics = commit_metrics[commit_metrics['commit_id'].isin(test_commit)]\n",
    "    \n",
    "    count_vect = CountVectorizer(min_df=3, ngram_range=(1,1))\n",
    "    count_vect.fit(train_code)\n",
    "    \n",
    "    train_feature, train_commit_id, new_train_label = get_combined_df(train_code, train_commit, train_label, train_commit_metrics,count_vect)\n",
    "    test_feature, test_commit_id, new_test_label = get_combined_df(test_code, test_commit, test_label, test_commit_metrics,count_vect)\n",
    "\n",
    "    percent_80 = int(len(new_train_label)*0.8)\n",
    "    \n",
    "    final_train_feature = train_feature[:percent_80]\n",
    "    final_train_commit_id = train_commit_id[:percent_80]\n",
    "    final_new_train_label = new_train_label[:percent_80]\n",
    "    \n",
    "    valid_feature = train_feature[percent_80:]\n",
    "    valid_commit_id = train_commit_id[percent_80:]\n",
    "    valid_label = new_train_label[percent_80:]\n",
    "\n",
    "    print('load data of',cur_proj, 'finish')\n",
    "    \n",
    "    bounds = [(1,20)]\n",
    "    result = differential_evolution(objective_func, bounds, args=(final_train_feature, final_new_train_label, \n",
    "                                                                  valid_feature, valid_label),\n",
    "                                   popsize=10, mutation=0.7, recombination=0.3,seed=0)\n",
    "    \n",
    "    smote = SMOTE(random_state=42, n_jobs=32, k_neighbors=int(np.round(result.x)))\n",
    "    train_feature_res, train_label_res = smote.fit_resample(final_train_feature, final_new_train_label)\n",
    "\n",
    "    clf = RandomForestClassifier(n_estimators=300, random_state=42, n_jobs=-1)\n",
    "    clf_name = 'RF'\n",
    "    trained_clf, pred_df = train_eval_model(clf, train_feature_res, train_label_res, \n",
    "                                       test_feature, new_test_label)\n",
    "    pred_df['test_commit'] = test_commit_id\n",
    "    pred_df.to_csv(data_path+cur_proj+'_'+clf_name+'_'+sampling_methods+'_prediction_result.csv')\n",
    "\n",
    "    model_path = model_path+cur_proj+'_'+clf_name+'_'+sampling_methods+'.pkl'\n",
    "    pickle.dump(trained_clf, open(model_path, 'wb'))\n",
    "\n",
    "    print('finished',cur_proj)\n",
    "    print('-'*100)\n",
    "\n",
    "    k_of_smote = result.x\n",
    "    best_AUC_of_obj_func = result.fun\n",
    "    \n",
    "    return k_of_smote, best_AUC_of_obj_func"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load data of openstack finish\n",
      "finished openstack\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "openstack_k_of_smote, openstack_best_AUC_of_obj_func = run_experiment('openstack')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The best k_neighbors of Openstack: 9\n"
     ]
    }
   ],
   "source": [
    "print('The best k_neighbors of Openstack:', openstack_k_of_smote)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load data of qt finish\n",
      "finished qt\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "qt_k_of_smote, qt_best_AUC_of_obj_func = run_experiment('qt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The best k_neighbors of Qt: 14\n"
     ]
    }
   ],
   "source": [
    "print('The best k_neighbors of Qt:', qt_k_of_smote)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RQ1-RQ2 result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "RF_data_dir = './data/'\n",
    "\n",
    "def get_recall_at_k_percent_effort(percent_effort, result_df_arg, real_buggy_commits):\n",
    "    cum_LOC_k_percent = (percent_effort/100)*result_df_arg.iloc[-1]['cum_LOC']\n",
    "    buggy_line_k_percent =  result_df_arg[result_df_arg['cum_LOC'] <= cum_LOC_k_percent]\n",
    "    buggy_commit = buggy_line_k_percent[buggy_line_k_percent['label']==1]\n",
    "    recall_k_percent_effort = len(buggy_commit)/float(len(real_buggy_commits))\n",
    "    \n",
    "    return recall_k_percent_effort\n",
    "\n",
    "def eval_metrics(result_df):\n",
    "    \n",
    "    pred = result_df['defective_commit_pred']\n",
    "    y_test = result_df['label']\n",
    "    \n",
    "    prec, rec, f1, _ = precision_recall_fscore_support(y_test,pred,average='binary') # at threshold = 0.5\n",
    "    tn, fp, fn, tp = confusion_matrix(y_test, pred, labels=[0, 1]).ravel()\n",
    "#     rec = tp/(tp+fn)\n",
    "    \n",
    "    FAR = fp/(fp+tn) # false alarm rate\n",
    "    dist_heaven = math.sqrt((pow(1-rec,2)+pow(0-FAR,2))/2.0) # distance to heaven\n",
    "    \n",
    "    AUC = roc_auc_score(y_test, result_df['defective_commit_prob'])\n",
    "\n",
    "    result_df['defect_density'] = result_df['defective_commit_prob']/result_df['LOC'] # predicted defect density\n",
    "    result_df['actual_defect_density'] = result_df['label']/result_df['LOC'] #defect density\n",
    "\n",
    "    result_df = result_df.sort_values(by='defect_density',ascending=False)\n",
    "    actual_result_df = result_df.sort_values(by='actual_defect_density',ascending=False)\n",
    "    actual_worst_result_df = result_df.sort_values(by='actual_defect_density',ascending=True)\n",
    "\n",
    "    result_df['cum_LOC'] = result_df['LOC'].cumsum()\n",
    "    actual_result_df['cum_LOC'] = actual_result_df['LOC'].cumsum()\n",
    "    actual_worst_result_df['cum_LOC'] = actual_worst_result_df['LOC'].cumsum()\n",
    "\n",
    "    real_buggy_commits = result_df[result_df['label'] == 1]\n",
    "\n",
    "    label_list = list(result_df['label'])\n",
    "\n",
    "    all_rows = len(label_list)\n",
    "\n",
    "    # find Recall@20%Effort\n",
    "    cum_LOC_20_percent = 0.2*result_df.iloc[-1]['cum_LOC']\n",
    "    buggy_line_20_percent = result_df[result_df['cum_LOC'] <= cum_LOC_20_percent]\n",
    "    buggy_commit = buggy_line_20_percent[buggy_line_20_percent['label']==1]\n",
    "    recall_20_percent_effort = len(buggy_commit)/float(len(real_buggy_commits))\n",
    "\n",
    "    # find Effort@20%Recall\n",
    "    buggy_20_percent = real_buggy_commits.head(math.ceil(0.2 * len(real_buggy_commits)))\n",
    "    buggy_20_percent_LOC = buggy_20_percent.iloc[-1]['cum_LOC']\n",
    "    effort_at_20_percent_LOC_recall = int(buggy_20_percent_LOC) / float(result_df.iloc[-1]['cum_LOC'])\n",
    "    \n",
    "    # find P_opt\n",
    "    percent_effort_list = []\n",
    "    predicted_recall_at_percent_effort_list = []\n",
    "    actual_recall_at_percent_effort_list = []\n",
    "    actual_worst_recall_at_percent_effort_list = []\n",
    "    \n",
    "    for percent_effort in np.arange(10,101,10):\n",
    "        predicted_recall_k_percent_effort = get_recall_at_k_percent_effort(percent_effort, result_df, real_buggy_commits)\n",
    "        actual_recall_k_percent_effort = get_recall_at_k_percent_effort(percent_effort, actual_result_df, real_buggy_commits)\n",
    "        actual_worst_recall_k_percent_effort = get_recall_at_k_percent_effort(percent_effort, actual_worst_result_df, real_buggy_commits)\n",
    "        \n",
    "        percent_effort_list.append(percent_effort/100)\n",
    "        \n",
    "        predicted_recall_at_percent_effort_list.append(predicted_recall_k_percent_effort)\n",
    "        actual_recall_at_percent_effort_list.append(actual_recall_k_percent_effort)\n",
    "        actual_worst_recall_at_percent_effort_list.append(actual_worst_recall_k_percent_effort)\n",
    "\n",
    "    p_opt = 1 - ((auc(percent_effort_list, actual_recall_at_percent_effort_list) - \n",
    "                 auc(percent_effort_list, predicted_recall_at_percent_effort_list)) /\n",
    "                (auc(percent_effort_list, actual_recall_at_percent_effort_list) -\n",
    "                auc(percent_effort_list, actual_worst_recall_at_percent_effort_list)))\n",
    "    \n",
    "    return prec, rec, f1, AUC, FAR, dist_heaven, recall_20_percent_effort, effort_at_20_percent_LOC_recall, p_opt\n",
    "\n",
    "def eval_result(proj_name,sampling_method = 'DE_SMOTE_min_df_3'):\n",
    "    \n",
    "    RF_result = pd.read_csv(RF_data_dir+proj_name+'_RF_'+sampling_method+'_prediction_result.csv')\n",
    "    \n",
    "    RF_result.columns = ['Unnamed', 'defective_commit_prob','defective_commit_pred','label','test_commit'] # for new result\n",
    "\n",
    "    test_code, test_commit, test_label = prepare_data(proj_name, mode='test',\n",
    "                                                              remove_python_common_tokens=remove_python_common_tokens)\n",
    "\n",
    "    # get LOC of each commit\n",
    "    RF_LOC = [len(code.splitlines()) for code in test_code]\n",
    "    RF_df = pd.DataFrame()\n",
    "    RF_df['commit_id'] = test_commit\n",
    "    RF_df['LOC'] = RF_LOC\n",
    "\n",
    "    RF_result = pd.merge(RF_df, RF_result,how='inner',left_on = 'commit_id', right_on='test_commit')\n",
    "    prec, rec, f1, auc, FAR, dist_heaven, recall_20_percent_effort, effort_at_20_percent_LOC_recall,p_opt = eval_metrics(RF_result)\n",
    "    \n",
    "    \n",
    "    print('Precision: {:.2f}, Recall: {:.2f}, F1: {:.2f}, AUC: {:.2f}, FAR: {:.2f}, d2h: {:.2f}, PCI@20%LOC: {:.2f}, Effort@20%Recall: {:.2f}, POpt: {:.2f}'.format(prec, rec, f1, auc, FAR, dist_heaven, recall_20_percent_effort, effort_at_20_percent_LOC_recall,p_opt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.43, Recall: 0.26, F1: 0.33, AUC: 0.83, FAR: 0.05, d2h: 0.52, PCI@20%LOC: 0.56, Effort@20%Recall: 0.04, POpt: 0.82\n"
     ]
    }
   ],
   "source": [
    "eval_result('openstack')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.43, Recall: 0.17, F1: 0.24, AUC: 0.82, FAR: 0.02, d2h: 0.59, PCI@20%LOC: 0.70, Effort@20%Recall: 0.02, POpt: 0.89\n"
     ]
    }
   ],
   "source": [
    "eval_result('qt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RQ3 result\n",
    "\n",
    "note: the_best_k_neighbors is obtained from model training phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_train_time(cur_proj, the_best_k_neighbors):\n",
    "    data_path = './data/'\n",
    "    model_path = './final_model/'\n",
    "        \n",
    "    train_code, train_commit, train_label = prepare_data(cur_proj, mode='train',\n",
    "                                                                  remove_python_common_tokens=remove_python_common_tokens)\n",
    "\n",
    "    commit_metrics = load_change_metrics_df(cur_proj)\n",
    "    train_commit_metrics = commit_metrics[commit_metrics['commit_id'].isin(train_commit)]\n",
    "    \n",
    "    count_vect = CountVectorizer(min_df=3, ngram_range=(1,1))\n",
    "    count_vect.fit(train_code)\n",
    "    \n",
    "    print('fit countvectorizer finished')\n",
    "    \n",
    "    train_feature, train_commit_id, new_train_label = get_combined_df(train_code, train_commit, train_label, train_commit_metrics,count_vect)\n",
    "\n",
    "    percent_80 = int(len(new_train_label)*0.8)\n",
    "    \n",
    "    final_train_feature = train_feature[:percent_80]\n",
    "    final_train_commit_id = train_commit_id[:percent_80]\n",
    "    final_new_train_label = new_train_label[:percent_80]\n",
    "    \n",
    "    smote = SMOTE(random_state=42, n_jobs=1, k_neighbors=the_best_k_neighbors)\n",
    "    \n",
    "    train_feature_res, train_label_res = smote.fit_resample(final_train_feature, final_new_train_label)\n",
    "\n",
    "    clf = RandomForestClassifier(n_estimators=300, random_state=42, n_jobs=-1)\n",
    "    clf_name = 'RF'\n",
    "    \n",
    "    start = time.time()\n",
    "    \n",
    "    clf.fit(train_feature_res, train_label_res)\n",
    "    \n",
    "    end = time.time()\n",
    "    \n",
    "    train_time = end-start\n",
    "    print('train time of {} is {:.3f} secs'.format(cur_proj,train_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit countvectorizer finished\n",
      "train time of openstack is 35.702 secs\n"
     ]
    }
   ],
   "source": [
    "check_train_time('openstack', 9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit countvectorizer finished\n",
      "train time of qt is 174.545 secs\n"
     ]
    }
   ],
   "source": [
    "check_train_time('qt', 14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python_Oat",
   "language": "python",
   "name": "env_oat"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}