Testing_net_attacks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train=pd.read_csv('train_data.csv')\n",
    "data_test=pd.read_csv('test_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of the Training set:  (169307, 43)\n",
      "The shape of the Testing set:  (91166, 42)\n"
     ]
    }
   ],
   "source": [
    "print(\"The shape of the Training set: \", data_train.shape)\n",
    "print(\"The shape of the Testing set: \", data_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_total=data_train.drop('target',axis=1)\n",
    "X_total=X_total.append(data_test)\n",
    "X_test=data_test[['connection_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train=data_train[['connection_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_total_num=X_total.drop(['connection_id','cat_1','cat_2','cat_3','cat_4','cat_5','cat_6','cat_7','cat_8','cat_9','cat_10','cat_11','cat_12','cat_13','cat_14','cat_15','cat_16','cat_17','cat_18','cat_19','cat_20','cat_21','cat_22','cat_23'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(260473, 42)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_total.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_total_num_scld=StandardScaler().fit_transform(X_total_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca=PCA()\n",
    "X_total_num_trf=pca.fit_transform(X_total_num_scld)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmgAAAFACAYAAAAI+ICPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYVdWZ6P/vCxIRERVBU4oGtG0FFFBLHIM2EYdoizOO\nVzFBkzjGXwYTOy16SV9vxzZGzXWIAxjFOQrakTii7YhViMqgYCsaaFTiAEKAFLB+f5xNpYA6xQE5\nA3W+n+c5z9nzfvdin/J1rb3XipQSkiRJqhxtyh2AJEmSVmaCJkmSVGFM0CRJkiqMCZokSVKFMUGT\nJEmqMCZokiRJFcYETZIkqcKYoEmSJFUYEzRJkqQKs1G5A/gqunTpkrp3717uMCRJktaovr7+Lyml\nroVsu0EnaN27d6eurq7cYUiSJK1RRHxQ6LY2cUqSJFUYEzRJkqQKY4ImSZJUYTboZ9Ca09DQwKxZ\ns1i8eHG5Q5FKon379nTr1o127dqVOxRJ0nrS6hK0WbNmsdlmm9G9e3ciotzhSEWVUuLTTz9l1qxZ\n9OjRo9zhSJLWk1bXxLl48WK22morkzNVhYhgq622ssZYklqZVpegASZnqire75LU+rTKBE2SJGlD\nVrQELSJuj4hPImJyk2WdI+LJiJiRfW/ZZN3PIuLdiHgnIg4rVlytzcyZM9ltt93WuM3o0aMb5+vq\n6rjwwguLHdpa6dix4xq32X///dfLuQops3W1vmKUJFW3YtagjQQOX2XZpcDTKaWdgaezeSKiF3Ay\n0Dvb5/9FRNsixlZVVk3Qamtrue6668oY0bp56aWXyh1CXkuXLgUqO0ZJ0oajaG9xppSej4juqywe\nDBycTY8CxgM/zZbfm1JaArwfEe8C/YGXixVfMd15551cffXVRAR9+vTh97//PWeddRZHHXUUJ5xw\nApCrMVqwYAHjx4/n8ssvZ4sttuCtt97ipJNOYvfdd+c3v/kNixYt4pFHHmGnnXbKu39TM2fO5Iwz\nzmDhwoUA3HDDDey///5ceumlTJs2jX79+nHmmWeyxx57cPXVVzN27Fh23HFHJk2axBZbbAHAzjvv\nzAsvvECbNm343ve+x4cffgjAtddeywEHHLDS+ZYtW8all17K+PHjWbJkCeeddx7nnnsuDz/8MDfc\ncANPPfUUH330EQcddBDPP/8848aN4+GHH2bevHnMnj2b008/ncsvv3ylYy5YsIDBgwfz+eef09DQ\nwIgRIxg8ePBqZTZ8+HC6dOnC5MmT2WuvvbjrrruICOrr67nkkktYsGABXbp0YeTIkdTU1FBfX8/Z\nZ58NwKGHHtrsv9vJJ5/MGWecwZFHHgnQWOa1tbXNluv48eP5xS9+wZZbbsnbb7/N9OnTG2PMdx0z\nZ87kiCOO4MADD+Sll15iu+22Y8yYMWyyySa8++67fO9732Pu3Lm0bduWBx54gJ122olf/epX3H//\n/SxZsoRjjz2WK664Yh3uSunvRr/6IWMmzS53GFLR9dq2E5f/c+9yh7FOSt3NxjYppTnZ9EfANtn0\ndsArTbablS1bTUScA5wDsMMOO7R4sisencLU/5n/VeJdzZr+sadMmcKIESN46aWX6NKlC5999tka\nj/nGG28wbdo0OnfuzI477sh3v/tdJkyYwG9+8xuuv/56rr322oJi23rrrXnyySdp3749M2bM4JRT\nTqGuro6rrrqKq6++msceewyA8ePHA9CmTRsGDx7Mww8/zNChQ3n11Vf5xje+wTbbbMOpp57KD3/4\nQw488EA+/PBDDjvsMKZNm7bS+W677TY233xzXnvtNZYsWcIBBxzAoYceyrHHHstDDz3Eb3/7W8aN\nG8cVV1zB17/+dQAmTJjA5MmT6dChA3vvvTdHHnkktbW1jcds3749Dz/8MJ06deIvf/kL++67L0cf\nffRqD8K//vrrTJkyhW233ZYDDjiAF198kX322YcLLriAMWPG0LVrV+677z4uu+wybr/9doYOHcoN\nN9zAgAED+PGPf9xs+Q0ZMoT777+fI488kr/97W88/fTT3HjjjaSUmi1XgIkTJzJ58uTVurjIdx0A\nM2bM4J577uF3v/sdJ510Eg899BCnn346p512GpdeeinHHnssixcvZvny5TzxxBPMmDGDCRMmkFLi\n6KOP5vnnn2fAgAEF3RPaMJQ6YXr1/dzfpX16dC7ZOSWtnbL1g5ZSShGR1mG/W4BbAGpra9d6/2J7\n5plnOPHEE+nSpQsAnTuv+Q/g3nvvTU1NDQA77bRTYw3P7rvvzrPPPlvwuRsaGjj//POZNGkSbdu2\nZfr06WvcZ8iQIVx55ZUMHTqUe++9lyFDhgDw1FNPMXXq1Mbt5s+fz4IFC1Z6VuyJJ57gzTff5MEH\nHwRg3rx5zJgxgx49enD99dez2267se+++3LKKac07jNo0CC22morAI477jheeOGFlRK0lBI///nP\nef7552nTpg2zZ8/m448/bkzwVujfvz/dunUDoF+/fsycOZMtttiCyZMnM2jQICBXw1dTU8MXX3zB\nF1980ZjUnHHGGTz++OOrlcURRxzBRRddxJIlSxg3bhwDBgxgk002Yd68eXnLtX///s32P5bvOgB6\n9OhBv379ANhrr72YOXMmX375JbNnz+bYY48FcgneijJ+4okn2GOPPYBcDeOMGTNM0IqoHLVLpU6Y\n9unRmcH9tuPUfVr+n1xJ5VPqBO3jiKhJKc2JiBrgk2z5bGD7Jtt1y5Z9JZVUrbnRRhuxfPlyAJYv\nX87f/va3xnUbb7xx43SbNm0a59u0adP4bFNL+6/w61//mm222YY33niD5cuXN/5HviX77bcf7777\nLnPnzuWRRx7hX/7lXxrP8corr7R4jJQS119/PYcdtvo7HbNmzaJNmzZ8/PHHLF++nDZtco87rloT\ntur83Xffzdy5c6mvr6ddu3Z079692T6+mpZZ27ZtWbp0KSklevfuzcsvr9wy/sUXX6yhFHLat2/P\nwQcfzJ/+9Cfuu+8+Tj75ZKDlct10002bPVZL17Fq7IsWLcobU0qJn/3sZ5x77rkFXYO+ujGTZjN1\nznx61XQq2TlNmCStqtQJ2ljgTOCq7HtMk+WjI+IaYFtgZ2BCiWNbLwYOHMixxx7LJZdcwlZbbcVn\nn31G586d6d69O/X19Zx00kmMHTuWhoaGtTpuIfvPmzePbt260aZNG0aNGsWyZcsA2Gyzzfjyyy+b\nPW5ENMbbs2fPxtqtQw89lOuvv76xOXDSpEmNtT4rHHbYYdx4440MHDiQdu3aMX36dLbbbjs23nhj\nzj77bO655x5GjRrFNddcw49+9CMAnnzyST777DM22WQTHnnkEW6//fbVrmHrrbemXbt2PPvss3zw\nwQcFl9Euu+zC3Llzefnll9lvv/1oaGhg+vTp9O7dmy222IIXXniBAw88kLvvvjvvMYYMGcKtt95K\nXV0dI0eObLFcW7K217HZZpvRrVs3HnnkEY455hiWLFnCsmXLOOyww/jFL37BaaedRseOHZk9ezbt\n2rVj6623LrhctPZ61XTivnP3K3cYkqpYMbvZuIfcQ/67RMSsiPgOucRsUETMAA7J5kkpTQHuB6YC\n44DzUkpr/q9gBerduzeXXXYZBx10EH379uWSSy4BYNiwYTz33HP07duXl19+OW/NSz6F7P+DH/yA\nUaNG0bdvX95+++3Gbfr06UPbtm3p27cvv/71r1fbb8iQIdx1112NzZsA1113HXV1dfTp04devXpx\n0003rbbfd7/7XXr16sWee+7JbrvtxrnnnsvSpUv5t3/7N775zW9y4IEHcs0113Drrbc2Pr/Wv39/\njj/+ePr06cPxxx+/UvMmwGmnnUZdXR277747d955J7vuumvBZfS1r32NBx98kJ/+9Kf07duXfv36\nNb5Veccdd3DeeefRr18/UsrfMn7ooYfy3HPPccghh/C1r32txXJtybpcx+9//3uuu+46+vTpw/77\n789HH33EoYceyqmnnsp+++3H7rvvzgknnJA32ZYktR7R0n+sKl1tbW1a8bD2CtOmTaNnz55likgt\nGTlyJHV1ddxwww3lDqXV8b5ff4bcnGsitwZN0voWEfUppdo1b9kKB0uX1LqU+qH9Uj9/JknNcagn\nlcxZZ51l7ZnW2oqH9kulV00nBvdrtpcfSSoZa9AkVTwf2pdUbaxBkyRJqjAmaJIkSRXGBE2SJKnC\ntPpn0B6Y8sB6Pd6JvU9c4zb7779/Y/9bhRg/fnzjWJljx45l6tSpXHrppXm3/9d//VcGDBjAIYcc\nkvc466J79+7U1dU1DlO1vq064Htz8l3bujj44IO5+uqrV+tr7atanzFuaMoxDJJvVUqqRq0+QSuH\ntUnOVnX00Uc3Dqqdz5VXXrnOx690lX5ty5Ytq/gYi6kcwyD5VqWkamSCVgQdO3ZkwYIFjB8/nuHD\nh9OlSxcmT57MXnvtxV133UVEMG7cOC6++GI6dOjAgQce2Ljvis5cf/nLX9KnTx/ef/992rRpw8KF\nC9l111157733GDZsWGNNVL7jDB8+nI4dOzYOsbTbbrvx2GOP0b17d4455hj+/Oc/s3jxYi666CLO\nOeecFq/niSee4PLLL2fJkiXstNNO3HHHHSxbtoz+/fszduxYdtllF0455RQGDhzIsGHD6NixI8OG\nDeOJJ57g61//Ovfeey9du3Zd6ZhXXnkljz76KIsWLWL//ffn5ptvJiJWqmXr3r07Z555Jo8++igN\nDQ088MAD7LrrrixcuJALLriAyZMn09DQwPDhwxk8eDCLFi1i6NChvPHGG+y6667NjnE5btw4brvt\nNh54IFez2rTW8fvf/z6vvfYaixYt4oQTTuCKK64AcjWLQ4YM4cknn+QnP/kJ48aNa4wx33UcfPDB\n7LPPPjz77LN88cUX3HbbbXzzm99k2bJl/PSnP2XcuHG0adOGYcOGccEFF1BfX88ll1zCggUL6NKl\nCyNHjqSmpmbdbsAi841KSSo+n0Erstdff51rr72WqVOn8t577/Hiiy+yePFihg0bxqOPPkp9fT0f\nffTRavttvvnm9OvXj+eeew6Axx57jMMOO4x27do1blPIcZpz++23U19fT11dHddddx2ffvpp3m3/\n8pe/MGLECJ566ikmTpxIbW0t11xzDZtvvjk33HADZ511Fvfeey+ff/45w4YNA2DhwoXU1tYyZcoU\nDjrooMZEp6nzzz+f1157jcmTJ7No0aK8zbJdunRh4sSJfP/73+fqq68G4Je//CUDBw5kwoQJPPvs\ns/z4xz9m4cKF3HjjjXTo0IFp06ZxxRVXUF9fv9rxDjnkEF599VUWLlwIsNKg6L/85S+pq6vjzTff\n5LnnnuPNN99s3G+rrbZi4sSJjdsWch1Lly5lwoQJXHvttY1lcMsttzBz5kwmTZrEm2++yWmnnUZD\nQwMXXHABDz74IPX19Zx99tlcdtllef9NJEmtnwlakfXv379xoO1+/foxc+ZM3n77bXr06MHOO+9M\nRHD66ac3u++QIUO47777ALj33ntXGisTKPg4q7ruuuvo27cv++67L3/+85+ZMWNG3m1feeUVpk6d\nygEHHEC/fv0YNWpU48DfgwYNYvfdd+e8887j1ltvbdynTZs2jbGefvrpvPDCC6sd99lnn2WfffZh\n991355lnnmHKlCnNnv+4444DYK+99mLmzJlArkbvqquuol+/fhx88MEsXryYDz/8kOeff76xDPr0\n6UOfPn1WO95GG23E4YcfzqOPPsrSpUv5z//8TwYPHgzA/fffz5577skee+zBlClTmDp1auN+q5Z9\nIdfRXOxPPfUU5557LhttlKu87ty5M++88w6TJ09m0KBB9OvXjxEjRjBr1qxmzydJqg42cRbZxhtv\n3Djdtm1bli5dWvC+Rx99ND//+c/57LPPqK+vZ+DAgQXvu9FGG7F8+fLG+cWLFwO5Jr2nnnqKl19+\nmQ4dOjQmOPmklBg0aBD33HPPauuWL1/OtGnT6NChA59//jndunVr9hgRsdL84sWL+cEPfkBdXR3b\nb789w4cPzxvDivJrWnYpJR566CF22WWXFkogv5NPPpkbbriBzp07U1tby2abbcb777/P1VdfzWuv\nvcaWW27JWWedtVJMzQ2QvqbraC725qSU6N27Ny+//PI6XY8kqfWxBq0Mdt11V2bOnMl///d/AzSb\n/EDuWba9996biy66iKOOOoq2bdsWfJzu3bszceJEACZOnMj7778PwLx589hyyy3p0KEDb7/9Nq+8\n8kqLse677768+OKLvPvuu0Cu+XL69OkA/PrXv6Znz56MHj2aoUOH0tDQAOQStwcffBCA0aNHr/Rs\nHPw9WezSpQsLFixo3LZQhx12GNdffz0pJSDXjAwwYMAARo8eDcDkyZNXaqJs6qCDDmLixIn87ne/\na2yynD9/Pptuuimbb745H3/8MY8//vga41iX6xg0aBA333xzY8L22WefscsuuzB37tzGBK2hoSFv\njaIkqTq0+hq0QrrFKLX27dtzyy23cOSRR9KhQwe++c1v8uWXXza77ZAhQzjxxBMZP378Wh3n+OOP\n584776R3797ss88+/OM//iMAhx9+ODfddBM9e/Zkl112Yd99920x1q5duzJy5EhOOeUUlixZAsCI\nESNIKXHrrbcyYcIENttsMwYMGMCIESO44oor2HTTTZkwYQIjRoxg6623bmymXWGLLbZg2LBh7Lbb\nbnz9619n7733Xqvy+8UvfsHFF19Mnz59WL58OT169Gh8yH/o0KH07NmTnj17stdeezW7f9u2bTnq\nqKMYOXIko0aNAqBv377sscce7Lrrrmy//fYccMABa4xjXa7ju9/9LtOnT6dPnz60a9eOYcOGcf75\n5/Pggw9y4YUXMm/ePJYuXcrFF19M796916pcJEmtR6yohdgQ1dbWprq6upWWTZs2jZ49e5YpIsHf\n32JV6ZTqvh9yc66Wz7c4JWntRUR9SqmgzjlbfQ2a1Jp9umAJXyxqYO6XSxh+c/GfYbPTWEkqDZ9B\n03pn7VnpfLGogcUNy0p2PjuNlaTSaJU1aCml1d4clFqrjTdqQ9fNNua+c/uVOxRJ0nrS6mrQ2rdv\nz6effsqG/GydVKiUEou/nEf79u3LHYokaT1qdTVo3bp1Y9asWcydO7fcoUhF98mXS5j71+UMPmD3\ncociSVqPWl2C1q5dO3r06FHuMKSSWPFiwAkHt1vDlpKkDUmra+KUJEna0JmgSZIkVRgTNEmSpApj\ngiZJklRhTNAkSZIqTKt7i1Mql9GvfsiYSbNLek6HXpKk1skaNGk9GTNpNlPnzC/pOR16SZJaJ2vQ\npPWoV00n7jt3v3KHIUnawFmDJkmSVGFM0CRJkiqMCZokSVKFMUGTJEmqMCZokiRJFcYETZIkqcKY\noEmSJFUYEzRJkqQKY4ImSZJUYRxJQK1WqcfGdFxMSdL6Yg2aWq1Sj43puJiSpPXFGjS1ao6NKUna\nEFmDJkmSVGHKkqBFxA8jYkpETI6IeyKifUR0jognI2JG9r1lOWKTJEkqt5InaBGxHXAhUJtS2g1o\nC5wMXAo8nVLaGXg6m5ckSao65Wri3AjYJCI2AjoA/wMMBkZl60cBx5QpNkmSpLIqeYKWUpoNXA18\nCMwB5qWUngC2SSnNyTb7CNim1LFJkiRVgnI0cW5JrrasB7AtsGlEnN50m5RSAlKe/c+JiLqIqJs7\nd27R45UkSSq1cjRxHgK8n1Kam1JqAP4A7A98HBE1ANn3J83tnFK6JaVUm1Kq7dq1a8mCliRJKpVy\nJGgfAvtGRIeICOBbwDRgLHBmts2ZwJgyxCZJklR2Je+oNqX0akQ8CEwElgKvA7cAHYH7I+I7wAfA\nSaWOTZIkqRKUZSSBlNLlwOWrLF5CrjZNkiSpqjmSgCRJUoUxQZMkSaowJmiSJEkVxgRNkiSpwpig\nSZIkVRgTNEmSpApTlm42VH1Gv/ohYybNLuk5p86ZT6+aTiU9pyRJ68Maa9AioltEPBwRcyPik4h4\nKCK6lSI4tR5jJs1m6pz5JT1nr5pODO63XUnPKUnS+lBIDdodwGjgxGz+9GzZoGIFpdapV00n7jt3\nv3KHIUlSxSvkGbSuKaU7UkpLs89IwFHKJUmSiqSQBO3TiDg9Itpmn9OBT4sdmCRJUrUqJEE7m9zA\n5R8Bc4ATgKHFDEqSJKmarfEZtJTSB8DRJYhFkiRJtJCgRcRPUkr/HhHXA2nV9SmlC4samSRJUpVq\nqQZtWvZdV4pAJEmSlJM3QUspPZpN/jWl9EDTdRFxYjO7SJIkaT0o5CWBnxW4TJIkSetBS8+gHQF8\nG9guIq5rsqoTsLTYgUmSJFWrlp5B+x9yz58dDdQ3Wf4l8MNiBiVJklTNWnoG7Q3gjYgYnVJqKGFM\nkiRJVa2QsTi7R8T/AXoB7VcsTCntWLSoJEmSqlghLwncAdxI7rmzfwLuBO4qZlCSJEnVrJAEbZOU\n0tNApJQ+SCkNB44sbliSJEnVq5AmziUR0QaYERHnA7OBjsUNS5IkqXoVUoN2EdABuBDYCzgdOLOY\nQUmSJFWzFmvQIqItMCSl9CNgATC0JFFJkiRVsRYTtJTSsog4sFTBqHRGv/ohYybNLtn5ps6ZT6+a\nTiU7nyRJG7JCnkF7PSLGAg8AC1csTCn9oWhRqejGTJpd0qSpV00nBvfbriTnkiRpQ1dIgtYe+BQY\n2GRZAkzQNnC9ajpx37n7lTsMSZK0ijUmaCklnzuTJEkqoULe4pQkSVIJmaBJkiRVGBM0SZKkCrPG\nBC0itomI2yLi8Wy+V0R8p/ihSZIkVadCatBGAn8Cts3mpwMXFysgSZKkaldIgtYlpXQ/sBwgpbQU\nWFbUqCRJkqpYIQnawojYilzfZ0TEvsC8okYlSZJUxQrpqPYSYCywU0S8CHQFTihqVJIkSVWskI5q\nJ0bEQcAuQADvpJQaih6ZJElSlSrkLc7zgI4ppSkppclAx4j4QfFDkyRJqk6FPIM2LKX0xYqZlNLn\nwLDihSRJklTdCknQ2kZErJiJiLbA14oXkiRJUnUr5CWBccB9EXFzNn9utkySJElFUEiC9lNySdn3\ns/kngVu/ykkjYovsGLuR677jbOAd4D6gOzATOClrTpUkSaoqa2ziTCktTyndmFI6IfvcnFL6qh3V\n/gYYl1LaFegLTAMuBZ5OKe0MPJ3NS5IkVZ1C3uI8ICKejIjpEfFeRLwfEe+t6wkjYnNgAHAbQErp\nb9lLCIOBUdlmo4Bj1vUckiRJG7JCmjhvA34I1LN+hnjqAcwF7oiIvtlxLwK2SSnNybb5CNimuZ0j\n4hzgHIAddthhPYQjSZJUWQp5i3NeSunxlNInKaVPV3y+wjk3AvYEbkwp7QEsZJXmzJRSIhtaalUp\npVtSSrUppdquXbt+hTAkSZIqUyE1aM9GxK+APwBLVixMKU1cx3POAmallF7N5h8kl6B9HBE1KaU5\nEVEDfLKOx9/gjH71Q8ZMml3Sc06dM59eNZ1Kek5JklSYQhK0fbLv2ibLEjBwXU6YUvooIv4cEbuk\nlN4BvgVMzT5nAldl32PW5fgbojGTZpc8YepV04nB/bYr2fkkSVLhChmL85+KcN4LgLsj4mvAe8BQ\ncs2t90fEd4APgJOKcN6K1aumE/edu1+5w5AkSRWgkBo0IuJIoDfQfsWylNKV63rSlNIkVq6RW+Fb\n63pMSZKk1qKQbjZuAoaQq/UK4ETgG0WOS5IkqWoV8hbn/iml/wV8nlK6AtgP+MfihiVJklS9CknQ\nFmXff42IbYEGoKZ4IUmSJFW3Qp5BeywbO/NXwERyb3B+pbE4JUmSlF8hb3H+72zyoYh4DGifUppX\n3LAkSZKqV94ELSIGppSeiYjjmllHSukPxQ1NkiSpOrVUg3YQ8Azwz82sS+RGFpAkSdJ6ljdBSyld\nHhFtgMdTSveXMCZJkqSq1uJbnCml5cBPShSLJEmSKKybjaci4kcRsX1EdF7xKXpkkiRJVaqQbjaG\nZN/nNVmWgB3XfziSJEkqpJuNHqUIRJIkSTmFDpa+G9CLlQdLv7NYQUmSJFWzNSZoEXE5cDC5BO2P\nwBHAC4AJmiRJUhEU8pLACcC3gI9SSkOBvsDmRY1KkiSpihU0WHrW3cbSiOgEfAJsX9ywJEmSqlch\nz6DVZYOl/w6oBxYALxc1KkmSpCpWyFucP8gmb4qIcUCnlNKbxQ1LkiSpeq2xiTMixkbEqRGxaUpp\npsmZJElScRXyDNp/AAcCUyPiwYg4ISLar2knSZIkrZtCmjifA56LiLbAQGAYcDvQqcixSZIkVaVC\nO6rdBPhncsM+7QmMKmZQkiRJ1ayQjmrvB/oD44AbgOeybjckSZJUBIXUoN0GnJJSWlbsYCRJklTY\nM2h/KkUgkiRJyinkLU5JkiSVkAmaJElShcnbxBkRe7a0Y0pp4voPR5IkSS09g/Yf2Xd7oBZ4Awig\nD1AH7Ffc0CRJkqpT3ibOlNI/pZT+CZgD7JlSqk0p7QXsAcwuVYCSJEnVppBn0HZJKb21YialNBno\nWbyQJEmSqlsh/aC9GRG3Andl86cBDpguSZJUJIUkaEOB7wMXZfPPAzcWLSJJkqQqV0hHtYsj4ibg\njymld0oQkyRJUlVb4zNoEXE0MIncWJxERL+IGFvswCRJkqpVIS8JXE5usPQvAFJKk4AexQxKkiSp\nmhWSoDWklOatsiwVIxhJkiQV9pLAlIg4FWgbETsDFwIvFTcsSZKk6lVIDdoFQG9gCXAPMB+4uJhB\nSZIkVbNC3uL8K3BZ9pEkSVKRrTFBi4h/BH4EdG+6fUppYPHCkiRJql6FPIP2AHATcCuwrLjhSJIk\nqZAEbWlKyZEDJEmSSqSQlwQejYgfRERNRHRe8fmqJ46IthHxekQ8ls13jognI2JG9r3lVz2HJEnS\nhqiQBO1M4Mfkutaozz516+HcFwHTmsxfCjydUtoZeDqblyRJqjprTNBSSj2a+ez4VU4aEd2AI8k9\n17bCYGBUNj0KOOarnEOSJGlDlfcZtIgYmFJ6JiKOa259SukPX+G81wI/ATZrsmyblNKcbPojYJs8\ncZ0DnAOR8XWZAAAP0klEQVSwww47fIUQJEmSKlNLLwkcBDwD/HMz6xKwTglaRBwFfJJSqo+Ig5vb\nJqWUIqLZ4aRSSrcAtwDU1tY65JQkSWp18iZoKaXLs++h6/mcBwBHR8S3gfZAp4i4C/g4ImpSSnMi\nogb4ZD2fV5IkaYNQSDcbRMSR5IZ7ar9iWUrpynU5YUrpZ8DPsuMeDPwopXR6RPyK3AsJV2XfY9bl\n+JIkSRu6Nb4kEBE3AUPIjckZwInAN4oQy1XAoIiYARySzUuSJFWdQmrQ9k8p9YmIN1NKV0TEfwCP\nr4+Tp5TGA+Oz6U+Bb62P40qSJG3ICukHbVH2/deI2BZoAGqKF5IkSVJ1K6QG7bGI2AL4FTCR3Buc\nt7a8iyRJktbVGhO0lNL/ziYfyoZlap9SmlfcsCRJkqpXSx3VNttBbbbuq3ZUK0mSpDxaqkFrroPa\nFda5o1pJkiS1rKWOatd3B7WSJEkqQCH9oG0VEddFxMSIqI+I30TEVqUITpIkqRoV8hbnvcDzwPHZ\n/GnAfeQ6k1UTVzw6han/M3+t95s6Zz69ajoVISJJkrQhKiRBq2nyJifAiIgYUqyAKtEDUx4oaLsZ\nn7Zj7l8L6Voup2uHrgD0qunE4H7brVNskiSp9SkkQXsiIk4G7s/mTwD+VLyQNlyD925Yq+1P7L1f\nkSKRJEkbskKqe4YBo4El2ede4NyI+DIi1r49T5IkSS0qpKPazUoRiCRJknIKeYvzO6vMt42Iy4sX\nkiRJUnUrpInzWxHxx4ioiYjdgFcAa9UkSZKKpJAmzlOztzbfAhYCp6aUXix6ZJIkSVWqkCbOnYGL\ngIeAD4AzIqJDsQOTJEmqVoU0cT4K/CKldC5wEDADeK2oUUmSJFWxQvpB659Smg+QUkrAf0TEo8UN\nS5IkqXrlrUGLiJ8ApJTmR8SJq6w+q5hBSZIkVbOWmjhPbjL9s1XWHV6EWCRJkkTLCVrkmW5uXpIk\nSetJSwlayjPd3LwkSZLWk5ZeEuibjbUZwCZNxt0MoH3RI5MkSapSeRO0lFLbUgYiSZKknEL6QZMk\nSVIJmaBJkiRVGBM0SZKkCmOCJkmSVGFM0CRJkiqMCZokSVKFMUGTJEmqMCZokiRJFcYETZIkqcKY\noEmSJFWYlsbiVJk8MOWBoh37xN4nFu3YkiRp/bAGTZIkqcKYoEmSJFUYEzRJkqQKY4ImSZJUYUzQ\nJEmSKowJmiRJUoUxQZMkSaowJmiSJEkVpuQJWkRsHxHPRsTUiJgSERdlyztHxJMRMSP73rLUsUmS\nJFWCctSgLQX+v5RSL2Bf4LyI6AVcCjydUtoZeDqblyRJqjolT9BSSnNSShOz6S+BacB2wGBgVLbZ\nKOCYUscmSZJUCcr6DFpEdAf2AF4FtkkpzclWfQRsk2efcyKiLiLq5s6dW5I4JUmSSqlsCVpEdAQe\nAi5OKc1vui6llIDU3H4ppVtSSrUppdquXbuWIFJJkqTSKkuCFhHtyCVnd6eU/pAt/jgiarL1NcAn\n5YhNkiSp3MrxFmcAtwHTUkrXNFk1Fjgzmz4TGFPq2CRJkirBRmU45wHAGcBbETEpW/Zz4Crg/oj4\nDvABcFIZYpMkSSq7kidoKaUXgMiz+luljEWSJKkSOZKAJElShTFBkyRJqjAmaJIkSRXGBE2SJKnC\nmKBJkiRVGBM0SZKkCmOCJkmSVGFM0CRJkiqMCZokSVKFMUGTJEmqMCZokiRJFcYETZIkqcKYoEmS\nJFUYEzRJkqQKY4ImSZJUYUzQJEmSKowJmiRJUoUxQZMkSaowJmiSJEkVxgRNkiSpwpigSZIkVRgT\nNEmSpApjgiZJklRhTNAkSZIqzEblDkCV4YEpDxTluCf2PrEox5UkqTUzQVNZFCshBJNCSdKGzyZO\nSZKkCmMNmqqGzbiSpA2FNWiSJEkVxgRNkiSpwpigSZIkVRgTNEmSpApjgiZJklRhfItTKhL7epMk\nrStr0CRJkiqMNWhSK2Jfb5LUOliDJkmSVGGsQZO0znzOTpKKwxo0SZKkCmOCJkmSVGFs4pS0QfFF\nCEnVwARNklrgc3aSyqHimjgj4vCIeCci3o2IS8sdjyRJUqlVVIIWEW2B3wJHAL2AUyKiV3mjkiRJ\nKq1Ka+LsD7ybUnoPICLuBQYDU8salSSVkM/ZSaq0BG074M9N5mcB+5QpFkmqCj5nJ1WeSkvQ1igi\nzgHOyWYXRMQ75YynGV2Av5Q7iApl2bTM8snPssnPssnPsmmZ5ZNfscrmG4VuWGkJ2mxg+ybz3bJl\njVJKtwC3lDKotRERdSml2nLHUYksm5ZZPvlZNvlZNvlZNi2zfPKrhLKpqJcEgNeAnSOiR0R8DTgZ\nGFvmmCRJkkqqomrQUkpLI+J84E9AW+D2lNKUMoclSZJUUhWVoAGklP4I/LHccXwFFdv8WgEsm5ZZ\nPvlZNvlZNvlZNi2zfPIre9lESqncMUiSJKmJSnsGTZIkqeqZoEmSJFUYE7R1tKYxQyPnumz9mxGx\nZzniLLWI2D4ino2IqRExJSIuamabgyNiXkRMyj7/Wo5YyyEiZkbEW9l11zWzvirvG4CI2KXJPTEp\nIuZHxMWrbFM1905E3B4Rn0TE5CbLOkfEkxExI/veMs++rXpM4zxl86uIeDv73TwcEVvk2bfF32Br\nkKd8hkfE7Ca/nW/n2bca7537mpTLzIiYlGff0t47KSU/a/kh94bpfwM7Al8D3gB6rbLNt4HHgQD2\nBV4td9wlKpsaYM9sejNgejNlczDwWLljLVP5zAS6tLC+Ku+bZsqhLfAR8I1VllfNvQMMAPYEJjdZ\n9u/Apdn0pcD/zVN2Lf592tA/ecrmUGCjbPr/Nlc22boWf4Ot4ZOnfIYDP1rDflV576yy/j+Af62E\ne8catHXTOGZoSulvwIoxQ5saDNyZcl4BtoiImlIHWmoppTkppYnZ9JfANHJDeKkwVXnfNONbwH+n\nlD4odyDlklJ6HvhslcWDgVHZ9CjgmGZ2LeTv0watubJJKT2RUlqazb5CrqPzqpTn3ilEVd47K0RE\nACcB95Q0qDxM0NZNc2OGrpqEFLJNqxYR3YE9gFebWb1/1hTxeET0Lmlg5ZWApyKiPhu2bFVVf99k\nTib/H8lqvXcAtkkpzcmmPwK2aWYb7yE4m1xNdHPW9BtszS7Ifju352ker/Z755vAxymlGXnWl/Te\nMUFTUURER+Ah4OKU0vxVVk8Edkgp9QGuBx4pdXxldGBKqR9wBHBeRAwod0CVJhtF5GiguRG8q/ne\nWUnKtbnYT9IqIuIyYClwd55NqvU3eCO5pst+wBxyTXla2Sm0XHtW0nvHBG3drHHM0AK3aZUioh25\n5OzulNIfVl2fUpqfUlqQTf8RaBcRXUocZlmklGZn358AD5NrUmiqau+bJo4AJqaUPl51RTXfO5mP\nVzR5Z9+fNLNN1d5DEXEWcBRwWpbArqaA32CrlFL6OKW0LKW0HPgdzV93Nd87GwHHAffl26bU944J\n2ropZMzQscD/yt7K2xeY16RpotXK2vBvA6allK7Js83Xs+2IiP7k7sNPSxdleUTEphGx2Yppcg81\nT15ls6q8b1aR9/9iq/XeaWIscGY2fSYwppltqnJM44g4HPgJcHRK6a95tinkN9gqrfIs67E0f91V\nee9kDgHeTinNam5lWe6dcr9RsaF+yL1tN53cGy+XZcu+B3wvmw7gt9n6t4DacsdconI5kFyzy5vA\npOzz7VXK5nxgCrk3hF4B9i933CUqmx2za34ju37vm9XLaFNyCdfmTZZV5b1DLkmdAzSQexboO8BW\nwNPADOApoHO27bbAH5vsu9rfp9b0yVM275J7fmrF352bVi2bfL/B1vbJUz6/z/6mvEku6arx3smV\nTbZ85Iq/M022Leu941BPkiRJFcYmTkmSpApjgiZJklRhTNAkSZIqjAmaJElShTFBkyRJqjAmaJK+\nsohYFhGTImJyRDwQER3ybPfHiNhiHY6/bUQ8+BXim1kNHdpGxFkRsW2545D01ZmgSVofFqWU+qWU\ndgP+Rq7vskZZx7ttUkrfTil9sbYHTyn9T0rphPUVbCt2Frm+myRt4EzQJK1v/wX8Q0R0j4h3IuJO\ncj1ub7+iJitbNy0ifhcRUyLiiYjYBCAi/iEinoqINyJiYkTslG0/OVt/VkSMiYjxETEjIi5fceKI\neCQbyHhKIYMZR8Th2TneiIins2Wds+O8GRGvRESfbPnwiBgVEf8VER9ExHER8e8R8VZEjMuGOFtR\nW7di+YSI+IdsefeIeCY77tMRsUO2fGREXBcRL0XEexFxQpP4fhwRr2X7XNHkOKuVXbZfLXB3Vpu5\nSURcFRFTs/2vXg//tpJKxARN0noTufHsjiDXYznAzsD/Syn1Til9sMrmOwO/TSn1Br4Ajs+W350t\n7wvsT67X71X1z7bvA5wYEbXZ8rNTSnuRS1QujIitWoi1K7kxCY/PznVituoK4PWUG5D958CdTXbb\nCRhIbjD3u4BnU0q7A4uAI5tsNy9bfgNwbbbsemBUdty7geuabF9DbhSOo4CrsvgOzcqoP7kBrveK\nvw/OvFrZpZQeBOrIjUPZD+hAbkif3tk5R+QrC0mVxwRN0vqwSURMIpcgfEhuPFaAD1JKr+TZ5/2U\n0qRsuh7ono11t11K6WGAlNLi1Py4ik+mlD5NKS0C/kAuuYFcUrZiGKjtySUy+ewLPJ9Sej8712fZ\n8gPJDYtDSukZYKuI6JStezyl1EAuAW0LjMuWvwV0b3Lse5p875dN7weMzqZ/3yRmgEdSSstTSlOB\nbbJlh2af14GJwK5Nrme1smvm+uYBi4HbIuI4oNnxKSVVpo3KHYCkVmFRVmvTKBvTfGEL+yxpMr0M\n2GQtzrfqGHUpIg4mN+Dxfimlv0bEeKD9WhyzEEsAUkrLI6Ih/X2svOWs/Pc05Zlu8biZaPL9f1JK\nNzfdMCK6U0DZpZSWZgPKfws4gdw4pgMLiEVSBbAGTVLFSCl9CcyKiGMAImLjPG+EDsqeFdsEOAZ4\nEdgc+DxLznYlV0PWkleAARHRIztX52z5fwGnZcsOBv6SUpq/lpcypMn3y9n0S8DJ2fRp2Xla8ifg\n7IjomMWyXURsvYZ9vgQ2y7bvSG7Q+T8CPwT6rtUVSCora9AkVZozgJsj4kqggdyzYctX2WYC8BDQ\nDbgrpVQXEW8B34uIacA75BKwvFJKc7MXCf4QEW2AT4BBwHDg9oh4k1yz4JnrcA1bZvsvAU7Jll0A\n3BERPwbmAkPXEN8TEdETeDmrjVwAnE6uxiyfkcBNEbGI3LOAYyKiPbnauEvW4ToklUn8vYZekipf\nRJwF1KaUzi93LM2JiJnk4vtLuWORtOGyiVOSJKnCWIMmSZJUYaxBkyRJqjAmaJIkSRXGBE2SJKnC\nmKBJkiRVGBM0SZKkCvP/AzWw9L5NoZhmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2a441ae9160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "var=(pca.explained_variance_)\n",
    "tot=sum(var)\n",
    "var_exp = [(i/tot)*100 for i in sorted(var, reverse=True)] # Individual explained variance\n",
    "\n",
    "exp_var_ratio=pca.explained_variance_ratio_\n",
    "cum_var=np.cumsum(exp_var_ratio*100)\n",
    "\n",
    "# PLOT OUT THE EXPLAINED VARIANCES SUPERIMPOSED \n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.bar(range(18), var_exp, alpha=0.3333, align='center', label='individual explained variance', color = 'g')\n",
    "plt.step(range(18), cum_var, where='mid',label='cumulative explained variance')\n",
    "plt.ylabel('Explained variance ratio')\n",
    "plt.xlabel('Principal components')\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca=PCA(n_components=10)\n",
    "X_total_num_trf_10d=pca.fit_transform(X_total_num_scld)\n",
    "X_total_pca=pd.DataFrame(X_total_num_trf_10d,columns=['PC1','PC2','PC3','PC4','PC5','PC6','PC7','PC8','PC9','PC10'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "del X_total_num_trf\n",
    "del X_total_num_trf_10d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>connection_id</th>\n",
       "      <th>cont_1</th>\n",
       "      <th>cont_2</th>\n",
       "      <th>cont_3</th>\n",
       "      <th>cont_4</th>\n",
       "      <th>cont_5</th>\n",
       "      <th>cont_6</th>\n",
       "      <th>cont_7</th>\n",
       "      <th>cont_8</th>\n",
       "      <th>cont_9</th>\n",
       "      <th>...</th>\n",
       "      <th>cat_14</th>\n",
       "      <th>cat_15</th>\n",
       "      <th>cat_16</th>\n",
       "      <th>cat_17</th>\n",
       "      <th>cat_18</th>\n",
       "      <th>cat_19</th>\n",
       "      <th>cat_20</th>\n",
       "      <th>cat_21</th>\n",
       "      <th>cat_22</th>\n",
       "      <th>cat_23</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cxcon_1</td>\n",
       "      <td>0</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cxcon_4</td>\n",
       "      <td>0</td>\n",
       "      <td>520</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cxcon_7</td>\n",
       "      <td>0</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cxcon_10</td>\n",
       "      <td>0</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cxcon_13</td>\n",
       "      <td>0</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  connection_id  cont_1  cont_2  cont_3  cont_4  cont_5  cont_6  cont_7  \\\n",
       "0       cxcon_1       0    1032       0     0.0     0.0     0.0     0.0   \n",
       "1       cxcon_4       0     520       0     0.0     0.0     0.0     0.0   \n",
       "2       cxcon_7       0    1032       0     0.0     0.0     0.0     0.0   \n",
       "3      cxcon_10       0    1032       0     0.0     0.0     0.0     0.0   \n",
       "4      cxcon_13       0    1032       0     0.0     0.0     0.0     0.0   \n",
       "\n",
       "   cont_8  cont_9   ...    cat_14  cat_15  cat_16  cat_17  cat_18  cat_19  \\\n",
       "0     1.0     0.0   ...         0       0       0       0       0       0   \n",
       "1     1.0     0.0   ...         0       0       0       0       0       0   \n",
       "2     1.0     0.0   ...         0       0       0       0       0       0   \n",
       "3     1.0     0.0   ...         0       0       0       0       0       0   \n",
       "4     1.0     0.0   ...         0       0       0       0       0       0   \n",
       "\n",
       "   cat_20  cat_21  cat_22  cat_23  \n",
       "0     511     511     255     255  \n",
       "1     511     511     255     255  \n",
       "2     511     511     255     255  \n",
       "3     511     511     255     255  \n",
       "4     511     511     255     255  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_total.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_total_cat=pd.DataFrame(X_total,columns=['cat_1','cat_2','cat_3','cat_4','cat_5','cat_6','cat_7','cat_8','cat_9','cat_10','cat_11','cat_12','cat_13','cat_14','cat_15','cat_16','cat_17','cat_18','cat_19','cat_20','cat_21','cat_22','cat_23'],dtype=str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature:- cat_1 has 3 unique categories\n",
      "Feature:- cat_2 has 66 unique categories\n",
      "Feature:- cat_3 has 11 unique categories\n",
      "Feature:- cat_4 has 2 unique categories\n",
      "Feature:- cat_5 has 3 unique categories\n",
      "Feature:- cat_6 has 2 unique categories\n",
      "Feature:- cat_7 has 19 unique categories\n",
      "Feature:- cat_8 has 3 unique categories\n",
      "Feature:- cat_9 has 2 unique categories\n",
      "Feature:- cat_10 has 15 unique categories\n",
      "Feature:- cat_11 has 2 unique categories\n",
      "Feature:- cat_12 has 3 unique categories\n",
      "Feature:- cat_13 has 13 unique categories\n",
      "Feature:- cat_14 has 11 unique categories\n",
      "Feature:- cat_15 has 3 unique categories\n",
      "Feature:- cat_16 has 7 unique categories\n",
      "Feature:- cat_17 has 1 unique categories\n",
      "Feature:- cat_18 has 2 unique categories\n",
      "Feature:- cat_19 has 2 unique categories\n",
      "Feature:- cat_20 has 462 unique categories\n",
      "Feature:- cat_21 has 431 unique categories\n",
      "Feature:- cat_22 has 256 unique categories\n",
      "Feature:- cat_23 has 256 unique categories\n"
     ]
    }
   ],
   "source": [
    "for col_name in X_total_cat.columns:\n",
    "    if X_total_cat[col_name].dtype=='object':\n",
    "        unique_cat=len(X_total_cat[col_name].unique())\n",
    "        print(\"Feature:- {col_name} has {unique_cat} unique categories\".format(col_name=col_name,unique_cat=unique_cat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    260473\n",
      "Name: cat_17, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(X_total_cat['cat_17'].value_counts().sort_values(ascending=False).head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10    148841\n",
      "13     58964\n",
      "1      33121\n",
      "2       5301\n",
      "11      3751\n",
      "Name: cat_2, dtype: int64\n",
      "511    118998\n",
      "1       20849\n",
      "510     14282\n",
      "2        5809\n",
      "509      3247\n",
      "3        3167\n",
      "4        2652\n",
      "5        2213\n",
      "6        1875\n",
      "7        1739\n",
      "8        1456\n",
      "9        1371\n",
      "10       1289\n",
      "11       1199\n",
      "12       1106\n",
      "13        947\n",
      "14        888\n",
      "508       798\n",
      "15        777\n",
      "16        713\n",
      "Name: cat_20, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(X_total_cat['cat_2'].value_counts().sort_values(ascending=False).head(5))\n",
    "print(X_total_cat['cat_20'].value_counts().sort_values(ascending=False).head(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 5000 # Anything that occurs less than this will be removed.\n",
    "for col in X_total_cat.columns:\n",
    "    vc = X_total_cat[col].value_counts()\n",
    "    vals_to_remove = vc[vc <= threshold].index.values\n",
    "    X_total_cat[col].loc[X_total_cat[col].isin(vals_to_remove)] = 'other'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature:- cat_1 has 3 unique categories\n",
      "Feature:- cat_2 has 5 unique categories\n",
      "Feature:- cat_3 has 4 unique categories\n",
      "Feature:- cat_4 has 2 unique categories\n",
      "Feature:- cat_5 has 2 unique categories\n",
      "Feature:- cat_6 has 2 unique categories\n",
      "Feature:- cat_7 has 2 unique categories\n",
      "Feature:- cat_8 has 2 unique categories\n",
      "Feature:- cat_9 has 2 unique categories\n",
      "Feature:- cat_10 has 2 unique categories\n",
      "Feature:- cat_11 has 2 unique categories\n",
      "Feature:- cat_12 has 2 unique categories\n",
      "Feature:- cat_13 has 2 unique categories\n",
      "Feature:- cat_14 has 2 unique categories\n",
      "Feature:- cat_15 has 2 unique categories\n",
      "Feature:- cat_16 has 2 unique categories\n",
      "Feature:- cat_17 has 1 unique categories\n",
      "Feature:- cat_18 has 2 unique categories\n",
      "Feature:- cat_19 has 2 unique categories\n",
      "Feature:- cat_20 has 5 unique categories\n",
      "Feature:- cat_21 has 7 unique categories\n",
      "Feature:- cat_22 has 2 unique categories\n",
      "Feature:- cat_23 has 3 unique categories\n"
     ]
    }
   ],
   "source": [
    "for col_name in X_total_cat.columns:\n",
    "    if X_total_cat[col_name].dtype=='object':\n",
    "        unique_cat=len(X_total_cat[col_name].unique())\n",
    "        print(\"Feature:- {col_name} has {unique_cat} unique categories\".format(col_name=col_name,unique_cat=unique_cat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_total_cat.drop('cat_17',axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_total_dummies=pd.get_dummies(X_total_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_1_1</th>\n",
       "      <th>cat_1_2</th>\n",
       "      <th>cat_1_3</th>\n",
       "      <th>cat_2_1</th>\n",
       "      <th>cat_2_10</th>\n",
       "      <th>cat_2_13</th>\n",
       "      <th>cat_2_2</th>\n",
       "      <th>cat_2_other</th>\n",
       "      <th>cat_3_1</th>\n",
       "      <th>cat_3_6</th>\n",
       "      <th>...</th>\n",
       "      <th>cat_21_3</th>\n",
       "      <th>cat_21_4</th>\n",
       "      <th>cat_21_510</th>\n",
       "      <th>cat_21_511</th>\n",
       "      <th>cat_21_other</th>\n",
       "      <th>cat_22_255</th>\n",
       "      <th>cat_22_other</th>\n",
       "      <th>cat_23_1</th>\n",
       "      <th>cat_23_255</th>\n",
       "      <th>cat_23_other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   cat_1_1  cat_1_2  cat_1_3  cat_2_1  cat_2_10  cat_2_13  cat_2_2  \\\n",
       "0        0        0        1        0         1         0        0   \n",
       "1        0        0        1        0         1         0        0   \n",
       "2        0        0        1        0         1         0        0   \n",
       "3        0        0        1        0         1         0        0   \n",
       "4        0        0        1        0         1         0        0   \n",
       "\n",
       "   cat_2_other  cat_3_1  cat_3_6      ...       cat_21_3  cat_21_4  \\\n",
       "0            0        1        0      ...              0         0   \n",
       "1            0        1        0      ...              0         0   \n",
       "2            0        1        0      ...              0         0   \n",
       "3            0        1        0      ...              0         0   \n",
       "4            0        1        0      ...              0         0   \n",
       "\n",
       "   cat_21_510  cat_21_511  cat_21_other  cat_22_255  cat_22_other  cat_23_1  \\\n",
       "0           0           1             0           1             0         0   \n",
       "1           0           1             0           1             0         0   \n",
       "2           0           1             0           1             0         0   \n",
       "3           0           1             0           1             0         0   \n",
       "4           0           1             0           1             0         0   \n",
       "\n",
       "   cat_23_255  cat_23_other  \n",
       "0           1             0  \n",
       "1           1             0  \n",
       "2           1             0  \n",
       "3           1             0  \n",
       "4           1             0  \n",
       "\n",
       "[5 rows x 59 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_total_dummies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_total_dummies=X_total_dummies.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_total_dummies.drop('index',axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_total_dummied=pd.concat([X_total_pca,X_total_dummies],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "del X_total_dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cid=X_total[['connection_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_total=X_total.reset_index()\n",
    "X_total.drop('index',axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_total_dummied['connection_id']=X_total['connection_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(260473, 70)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_total_dummied.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train=pd.merge(X_train,X_total_dummied,how='left',on='connection_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(169307, 70)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_total_dummies=pd.concat([X_total_pca,X_total_dummies],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test=pd.merge(X_test,X_total_dummied,how='left',on='connection_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(91166, 70)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_train=pd.concat([X_train_pca,X_train],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train=data_train[['target']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier,ExtraTreesClassifier,GradientBoostingClassifier,AdaBoostClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix,precision_recall_curve,auc,roc_auc_score,roc_curve,recall_score,classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold,KFold\n",
    "\n",
    "num_train = X_train.shape[0]\n",
    "num_test = X_test.shape[0]\n",
    "SEED = 0 # for same randomized reproducibility\n",
    "NFOLDS = 5 # set folds for out-of-fold prediction\n",
    "skf = StratifiedKFold(n_splits=NFOLDS, random_state=SEED)\n",
    "#kf = KFold(n_splits= NFOLDS, random_state=SEED)\n",
    "\n",
    "# Class to extend the Sklearn classifier\n",
    "\n",
    "class Sklearn_Rep(object):\n",
    "    def __init__(self, clf, seed=0, params=None):\n",
    "        params['random_state'] = seed\n",
    "        self.clf = clf(**params)\n",
    "\n",
    "    def train(self, x_train, y_train):\n",
    "        self.clf.fit(x_train, y_train)\n",
    "\n",
    "    def predict(self, x):\n",
    "        return self.clf.predict(x)\n",
    "    \n",
    "    def fit(self,x,y):\n",
    "        return self.clf.fit(x,y)\n",
    "    \n",
    "    def feature_importances(self,x,y):\n",
    "        print(self.clf.fit(x,y).feature_importances_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_oof(clf, x_train, y_train, x_test):\n",
    "    oof_train = np.zeros((num_train,))\n",
    "    oof_test = np.zeros((num_test,))\n",
    "    oof_test_skf = np.empty((NFOLDS, num_test))\n",
    "\n",
    "    for i,(train_index, test_index) in enumerate(skf.split(x_train,y_train)):\n",
    "        x_tr = x_train[train_index]\n",
    "        y_tr = y_train[train_index]\n",
    "        x_te = x_train[test_index]\n",
    "\n",
    "        clf.train(x_tr, y_tr)\n",
    "\n",
    "        oof_train[test_index] = clf.predict(x_te)\n",
    "        oof_test_skf[i, :] = clf.predict(x_test)\n",
    "\n",
    "    #oof_test[:] = (oof_test_skf.mean(axis=0))\n",
    "    return oof_train, oof_test_skf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Put in our parameters for said classifiers\n",
    "# Random Forest parameters\n",
    "rf_params = {\n",
    "    'n_jobs': -1,\n",
    "    'n_estimators': 500,\n",
    "     #'warm_start': True, \n",
    "     #'max_features': 0.2,\n",
    "    'max_depth': 8,\n",
    "    #'min_samples_leaf': 2,\n",
    "    #'max_features' : 'sqrt',\n",
    "    #'verbose': 0,\n",
    "}\n",
    "\n",
    "# Extra Trees Parameters\n",
    "et_params = {\n",
    "    'n_jobs': -1,\n",
    "    'n_estimators':500,\n",
    "    #'max_features': 0.5,\n",
    "    'max_depth': 10,\n",
    "    #'min_samples_leaf': 2,\n",
    "    'verbose': 0,\n",
    "}\n",
    "\n",
    "# AdaBoost parameters\n",
    "ada_params = {\n",
    "    'n_estimators': 500,\n",
    "    'learning_rate' : 0.75\n",
    "}\n",
    "\n",
    "# xtreme Gradient Boosting parameters\n",
    "xgb_params = {'n_estimators': 1000,\n",
    " 'max_depth': 5,\n",
    " 'objective': 'multi:softprob',\n",
    " 'nthread': -1,\n",
    " 'scale_pos_weight':1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create objects that represent our models\n",
    "rf = Sklearn_Rep(clf=RandomForestClassifier, seed=SEED, params=rf_params)\n",
    "et = Sklearn_Rep(clf=ExtraTreesClassifier, seed=SEED, params=et_params)\n",
    "#ada = Sklearn_Rep(clf=AdaBoostClassifier, seed=SEED, params=ada_params)\n",
    "gb = Sklearn_Rep(clf=XGBClassifier, seed=SEED, params=xgb_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_cid=X_train['connection_id']\n",
    "X_train=X_train.drop('connection_id',axis=1)\n",
    "X_test_cid=X_test['connection_id']\n",
    "X_test=X_test.drop('connection_id',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = pd.factorize(Y_train['target'])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(169307,)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_oof_train, rf_oof_test_nfolds = get_oof(rf, X_train.values, Y_train.values.ravel(), X_test.values) # Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    98868\n",
       "2    36854\n",
       "1    33585\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train['target'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr=rf_oof_test_nfolds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "axis= 0\n",
    "u, indices = np.unique(arr, return_inverse=True)\n",
    "rf_oof_test=u[np.argmax(np.apply_along_axis(np.bincount, axis, indices.reshape(arr.shape),\n",
    "                                None, np.max(indices) + 1), axis=axis)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.  0.  0. ...,  0.  0.  1.]\n"
     ]
    }
   ],
   "source": [
    "print(rf_oof_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "et_oof_train, et_oof_test_nfolds = get_oof(et, X_train.values, Y_train.values.ravel(), X_test.values) # extratrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr=et_oof_test_nfolds\n",
    "axis= 0\n",
    "u, indices = np.unique(arr, return_inverse=True)\n",
    "et_oof_test=u[np.argmax(np.apply_along_axis(np.bincount, axis, indices.reshape(arr.shape),\n",
    "                                None, np.max(indices) + 1), axis=axis)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  0.,  0., ...,  0.,  0.,  1.],\n",
       "       [ 1.,  0.,  0., ...,  0.,  0.,  1.],\n",
       "       [ 1.,  0.,  0., ...,  0.,  0.,  1.],\n",
       "       [ 1.,  0.,  0., ...,  0.,  0.,  1.],\n",
       "       [ 1.,  0.,  0., ...,  0.,  0.,  1.]])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(et_oof_test_nfolds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_et=pd.DataFrame(et_oof_test,columns=['target_et'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target_et</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   target_et\n",
       "0        1.0\n",
       "1        0.0\n",
       "2        0.0\n",
       "3        0.0\n",
       "4        0.0"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions_et.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_rf=pd.DataFrame(rf_oof_test,columns=['target_rf'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_oof_train, xgb_oof_test_nfolds = get_oof(gb, X_train.values, Y_train.values.ravel(), X_test.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_oof_test_nfolds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr=xgb_oof_test_nfolds\n",
    "axis= 0\n",
    "u, indices = np.unique(arr, return_inverse=True)\n",
    "xgb_oof_test=u[np.argmax(np.apply_along_axis(np.bincount, axis, indices.reshape(arr.shape),\n",
    "                                None, np.max(indices) + 1), axis=axis)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(xgb_oof_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_xgb=pd.DataFrame(xgb_oof_test,columns=['target_xgb'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_xgb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eclf1 = VotingClassifier(estimators=[('et', et), ('rf', rf), ('gb', gb)], voting='hard')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbm = xgb.XGBClassifier(\n",
    " n_estimators= 1000,\n",
    " max_depth= 5,\n",
    " objective= 'multi:softprob',\n",
    " nthread= -1,\n",
    " scale_pos_weight=1).fit(X_train.values, Y_train.values.ravel())\n",
    "predictions = gbm.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_rf['target_rf'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('sample_submission.csv')\n",
    "sub['target'] = xgb_oof_test\n",
    "sub['target'] = sub['target'].astype(int)\n",
    "sub.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions=predictions_rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = [predictions_rf, predictions_et, predictions_xgb]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions=pd.concat(frames,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_final=predictions.mode(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_final.drop([1,2],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('sample_submission.csv')\n",
    "sub['target'] = pred_final[0]\n",
    "sub['target'] = sub['target'].astype(int)\n",
    "sub.to_csv('submission4.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}