JVAP_grid_search_and_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.metrics import accuracy_score,make_scorer,f1_score,classification_report,average_precision_score\n",
    "from sklearn.preprocessing import Normalizer,MinMaxScaler,StandardScaler,normalize\n",
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.set_option('max_columns',1000)\n",
    "pd.set_option('max_rows',1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#############################################\n",
    "# PANDAS HELPERS\n",
    "#############################################\n",
    "\n",
    "def remove_column_from_data_frame(col_to_remove, data_frame):\n",
    "\n",
    "    if col_to_remove in list(data_frame.columns):\n",
    "        data_frame.drop(col_to_remove, axis=1, inplace=True)\n",
    "\n",
    "        \n",
    "def remove_columns_from_data_frame(cols_to_remove, data_frame):\n",
    "\n",
    "    column_dict = {x: None for x in list(data_frame.columns)}\n",
    "\n",
    "    cols_to_remove = [x for x in cols_to_remove if x in column_dict]\n",
    "\n",
    "    data_frame.drop(labels=cols_to_remove, axis=1, inplace=True)\n",
    "    \n",
    "\n",
    "def remove_columns_like(column_pattern, data_frame):\n",
    "    \n",
    "    for column in list(data_frame.columns):\n",
    "        if column_pattern in column:\n",
    "            data_frame.drop(column, axis=1, inplace=True)\n",
    "\n",
    "\n",
    "def fill_nas(value, data_frame):\n",
    "    \n",
    "    data_frame.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#############################################\n",
    "# DATA RETRIEVAL HELPERS\n",
    "#############################################\n",
    "\n",
    "def get_data(n_rows=None):\n",
    "\n",
    "    if n_rows is not None:\n",
    "        df = pd.read_csv('final_feats_without_dummies_3.csv', low_memory=False, nrows=n_rows)\n",
    "        df_y = pd.read_csv('final_outs_3.csv', low_memory=False, nrows=n_rows)\n",
    "    else:\n",
    "        df = pd.read_csv('final_feats_without_dummies_3.csv', low_memory=False)\n",
    "        df_y = pd.read_csv('final_outs_3.csv', low_memory=False)\n",
    "    \n",
    "    \n",
    "    # Drop labels and a redundant column\n",
    "    remove_columns_from_data_frame(['Unnamed: 0', 'Unnamed: 0.1' 'dissent', 'dissentdummy'], df)\n",
    "    df,df_y=remove_bad_rows(df,df_y)\n",
    "    df=drop_unneeded_cols(df)\n",
    "    df=drop_dissent(df)\n",
    "    print df.shape\n",
    "    df=dummify(df)\n",
    "    print df.shape\n",
    "    # Extras -- for analysis\n",
    "    # CASE 1: REMOVE TOP 2\n",
    "    # CASE 2: REMOVE ALL 'DISS'\n",
    "    \n",
    "#     remove_columns_from_data_frame(['type', 'turnonthresh'], df)\n",
    "#     remove_columns_from_data_frame(['type1', 'last3'], df)\n",
    "#     remove_columns_like('diss', df)\n",
    "    \n",
    "    return df, df_y\n",
    "\n",
    "\n",
    "def get_x_y(n_rows=None):\n",
    "    \n",
    "    df, df_y = get_data(n_rows)\n",
    "\n",
    "    #fill_nas(0, df)\n",
    "    \n",
    "    return df.values, df_y.ix[:,0].values\n",
    "\n",
    "\n",
    "def get_columns(df):\n",
    "    \n",
    "    #df = pd.get_dummies(pd.read_csv('final_feats_without_dummies.csv', low_memory=False, nrows=2))\n",
    "    return list(df.columns)\n",
    "\n",
    "\n",
    "def print_report(y, y_pred):\n",
    "\n",
    "    print classification_report(y, y_pred)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#############################################\n",
    "# MODEL HELPERS\n",
    "#############################################\n",
    "\n",
    "def grid_search(X, y, clf, param_grid):\n",
    "    \n",
    "#     param_dict={'average': 'weighted'}\n",
    "    scorer = make_scorer(average_precision_score)\n",
    "\n",
    "\n",
    "    gridclf = GridSearchCV(clf, paramgrid, scoring=scorer, cv=3, verbose=1)\n",
    "\n",
    "    gridclf.fit(X, y)\n",
    "\n",
    "    print gridclf.best_params_\n",
    "    print gridclf.best_estimator_\n",
    "\n",
    "    print_report(y_test, gridclf.predict(X_test))\n",
    "    \n",
    "\n",
    "def get_top_n_feats(n, feat_arr, cols):\n",
    "    args=np.argsort(feat_arr)\n",
    "    assert len(feat_arr)==len(cols)\n",
    "    col_scores=col_scores=np.array(zip(cols,feat_arr))\n",
    "    return col_scores[args[-n:]].tolist()[::-1]\n",
    "\n",
    "    \n",
    "# def get_top_n(n, arr, col_names, prev_list=[]):\n",
    "    \n",
    "#     if n <= 0:\n",
    "#         return []\n",
    "    \n",
    "#     most_imp = -1\n",
    "#     most_imp_index = -1\n",
    "\n",
    "#     for i in range(len(arr)):\n",
    "\n",
    "#         if i in prev_list:\n",
    "#             continue\n",
    "\n",
    "#         if arr[i] > most_imp:\n",
    "#             most_imp = arr[i]\n",
    "#             most_imp_index = i\n",
    "\n",
    "#     prev_list.append(most_imp_index)\n",
    "\n",
    "#     return [ (col_names[most_imp_index], most_imp) ] + get_top_n(n - 1, arr, col_names, prev_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def drop_unneeded_cols(df):\n",
    "    del_cols = ['fileid','cite','vol','beginpg','endopin','endpage','docnum','priorpub','_merge','year',\n",
    "            'circuit','pseatno','decision_date','aatty_first_name','aatty_last_name','afirm_name',\n",
    "            'ratty_first_name','ratty_last_name','rname_of_first_listed_amicus_gro','rfirm_namew','decisiondatenew2',\n",
    "           'j1name','j2name','j3name','quartertoelect','pname','seatno','success','lsuc','ls1','ls2','ls3','lp',\n",
    "            'lp2','lp3','sseatno','congress','congreso','afirst_listed_amicus_group','yearquarter','name','Name','State','j',\n",
    "            'codej4','j4vote1','j4vote2','j4maj1','j4maj2','codej5','j5vote1','j5vote2','j5maj1','j5maj2',\n",
    "            'codej6','j6vote1','j6vote2','j6maj1','j6maj2','codej7','j7vote1','j7vote2','j7maj1','j7maj2',\n",
    "            'codej8','j8vote1','j8vote2','j8maj1','j8maj2','codej9','j9vote1','j9vote2','j9maj1','j9maj2',\n",
    "            'codej10','j10vote1','j10vote2','j10maj1','j10maj2','codej11','j11vote1','j11vote2','j11maj1','j11maj2',\n",
    "            'codej12','j12vote1','j12vote2','j12maj1','j12maj2','codej13','j13vote1','j13vote2','j13maj1','j13maj2',\n",
    "            'codej14','j14vote1','j14vote2','j14maj1','j14maj2','codej15','j15vote1','j15vote2','j15maj1','j15maj2','j16maj1','j16vote1']\n",
    "    df.drop(labels=del_cols,axis=1,inplace=True)\n",
    "    moredropcolumns=df.columns.tolist() # .tolist?\n",
    "    for i in moredropcolumns:\n",
    "        if len(pd.unique(df[i]))==1:\n",
    "            df.drop(labels=i,axis=1,inplace=True)\n",
    "    df.drop(labels=['casenum','j2vote1','j2vote2','j2maj1','direct1',\n",
    "                          'j2maj2','j3vote1','j3vote2','j3maj1','j3maj2','majvotes','ids'],axis=1,inplace=True)\n",
    "    return df\n",
    "    \n",
    "def dummify(df,max_uniq=100,sparse=True):\n",
    "    new_cols=df.columns\n",
    "    new_cols=new_cols.tolist()\n",
    "#     keep_cols=['j1score','j2score','j3score','popularpct','electoralpct','closerd','fartherd','dAds3','dF2Ads3',\n",
    "#            'dF1Ads3','dL1Ads3','dL2Ads3','dL3Ads3','dL4Ads3','dL5Ads3','logAds3','logL1Ads3','logL2Ads3','logF1Ads3',\n",
    "#           'logF2Ads3','decade2','propneg','likely_elev2','score','d12','d13','d23','sat_together_count']\n",
    "\n",
    "    float_cols=['j1score','j2score','j3score','popularpct','electoralpct','closerd','fartherd','dAds3','dF2Ads3',\n",
    "           'dF1Ads3','dL1Ads3','dL2Ads3','dL3Ads3','dL4Ads3','dL5Ads3','logAds3','logL1Ads3','logL2Ads3','logF1Ads3',\n",
    "          'logF2Ads3','decade2','propneg','likely_elev2','score','d12','d13','d23',\n",
    "           'judgecitations','experience','experiencetrun','age2trun','agego','assets','ba','liable',\n",
    "            'networth','totalcities','sat_together_count','keytotal','lengthopin','Wopinionlenght','Wtotalcites','age']\n",
    "\n",
    "    remove_for_now=['Ads3','F1Ads3','F2Ads3','L1Ads3','L2Ads3','L3Ads3','L4Ads3','L5Ads3','Unnamed: 0.1','appel1','appel2',\n",
    "               'citevol','codej3','id','usc2sect','usc1sect','age2','distjudg','respond1','respond2','yearb','pred','csb']\n",
    "\n",
    "    \n",
    "    #print 'Ads3' in df.columns\n",
    "    for x in remove_for_now:\n",
    "        if x in df.columns:\n",
    "            print \"dropped: \",x\n",
    "            df.drop(labels=[x],inplace=True,axis=1)\n",
    "    sum1=0\n",
    "    \n",
    "    dummy_cols=[]\n",
    "    for col in df.columns:\n",
    "        if col not in float_cols:\n",
    "            if len(pd.unique(df.ix[:,col]))>=max_uniq or (df.ix[:,col].dtype!='float64' and df.ix[:,col].dtype!='int64'): \n",
    "                sum1+= len(pd.unique(df.ix[:,col]))\n",
    "                dummy_cols.append(col)\n",
    "    print \"# of dummy columns: \",sum1\n",
    "    print df.shape\n",
    "    print dummy_cols\n",
    "    df2=pd.get_dummies(df,columns=dummy_cols,dummy_na=True,sparse=sparse)\n",
    "    df2.fillna(value=0,inplace=True)\n",
    "    #print df2.shape\n",
    "    return df2\n",
    "\n",
    "\n",
    "def remove_bad_rows(df_x,df_y):\n",
    "    \n",
    "    #remove rows where codej1==codej2\n",
    "#     df[df.codej1==df.codej2].index\n",
    "    \n",
    "    same_cols = df_x[df_x.codej1==df_x.codej2].index\n",
    "    df_x=df_x.drop(same_cols).reset_index(drop=True)\n",
    "    df_y=df_y.drop(same_cols).reset_index(drop=True)\n",
    "    \n",
    "    #remove rows where >3 judges occur\n",
    "#     pp = pd.read_csv('../raw/Votelevel_stuffjan2013.csv')\n",
    "#     qq=pp.groupby(by=['casenum']).count()\n",
    "#     pd.unique(qq.month)\n",
    "#     rr=qq[qq.month==6].reset_index()\n",
    "#     rr.shape\n",
    "    \n",
    "    #remove rows where codej2==null\n",
    "    #df[map(lambda x: not(x),pd.notnull(df.ix[:][\"codej2\"]).tolist())]\n",
    "    \n",
    "    nan_cols=df_x[map(lambda x: not(x),pd.notnull(df_x.ix[:][\"codej2\"]).tolist())].index\n",
    "    nan_cols.append(df_x[map(lambda x: not(x),pd.notnull(df_x.ix[:][\"codej1\"]).tolist())].index)\n",
    "    df_x=df_x.drop(nan_cols).reset_index(drop=True)\n",
    "    df_y=df_y.drop(nan_cols).reset_index(drop=True)\n",
    "    \n",
    "    return df_x,df_y\n",
    "\n",
    "def drop_dissent(df):\n",
    "    diss_list=[]\n",
    "    for col in df.columns:\n",
    "        if 'diss' in col or 'concur' or 'unan' in col:\n",
    "            #print col\n",
    "            diss_list.append(col)\n",
    "    df.drop(labels=diss_list,axis=1,inplace=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(111538, 0)\n",
      "# of dummy columns:  0\n",
      "(111538, 0)\n",
      "[]\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "No objects to concatenate",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-1a891aa96f1c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdf_x\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdf_y\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-4-65ce3e0aac27>\u001b[0m in \u001b[0;36mget_data\u001b[1;34m(n_rows)\u001b[0m\n\u001b[0;32m     19\u001b[0m     \u001b[0mdf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdrop_dissent\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 21\u001b[1;33m     \u001b[0mdf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdummify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     22\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m     \u001b[1;31m# Extras -- for analysis\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-6-acf781ac8a69>\u001b[0m in \u001b[0;36mdummify\u001b[1;34m(df, max_uniq, sparse)\u001b[0m\n\u001b[0;32m     53\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     54\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[0mdummy_cols\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 55\u001b[1;33m     \u001b[0mdf2\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_dummies\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdummy_cols\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdummy_na\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msparse\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msparse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     56\u001b[0m     \u001b[0mdf2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     57\u001b[0m     \u001b[1;31m#print df2.shape\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/share/apps/pandas/0.17.1/intel/lib/python2.7/site-packages/pandas-0.17.1-py2.7-linux-x86_64.egg/pandas/core/reshape.pyc\u001b[0m in \u001b[0;36mget_dummies\u001b[1;34m(data, prefix, prefix_sep, dummy_na, columns, sparse)\u001b[0m\n\u001b[0;32m   1069\u001b[0m                                     dummy_na=dummy_na, sparse=sparse)\n\u001b[0;32m   1070\u001b[0m             \u001b[0mwith_dummies\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdummy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1071\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconcat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwith_dummies\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1072\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1073\u001b[0m         result = _get_dummies_1d(data, prefix, prefix_sep, dummy_na,\n",
      "\u001b[1;32m/share/apps/pandas/0.17.1/intel/lib/python2.7/site-packages/pandas-0.17.1-py2.7-linux-x86_64.egg/pandas/tools/merge.pyc\u001b[0m in \u001b[0;36mconcat\u001b[1;34m(objs, axis, join, join_axes, ignore_index, keys, levels, names, verify_integrity, copy)\u001b[0m\n\u001b[0;32m    810\u001b[0m                        \u001b[0mkeys\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkeys\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlevels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlevels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnames\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnames\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    811\u001b[0m                        \u001b[0mverify_integrity\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverify_integrity\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 812\u001b[1;33m                        copy=copy)\n\u001b[0m\u001b[0;32m    813\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    814\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/share/apps/pandas/0.17.1/intel/lib/python2.7/site-packages/pandas-0.17.1-py2.7-linux-x86_64.egg/pandas/tools/merge.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, objs, axis, join, join_axes, keys, levels, names, ignore_index, verify_integrity, copy)\u001b[0m\n\u001b[0;32m    843\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    844\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobjs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 845\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'No objects to concatenate'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    846\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    847\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mkeys\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: No objects to concatenate"
     ]
    }
   ],
   "source": [
    "df_x,df_y = get_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X=df_x.values\n",
    "y=df_y.ix[:,2].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(111538, 0)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "labels ['Ads3' 'F1Ads3' 'F2Ads3' 'L1Ads3' 'L2Ads3' 'L3Ads3' 'L4Ads3' 'L5Ads3'\n 'Unnamed: 0.1' 'appel1' 'appel2' 'citevol' 'codej3' 'id' 'usc2sect'\n 'usc1sect' 'age2' 'distjudg' 'respond1' 'respond2' 'yearb' 'pred' 'csb'] not contained in axis",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-83-12a3c8358981>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m#############################################\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_x_y\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-79-65ce3e0aac27>\u001b[0m in \u001b[0;36mget_x_y\u001b[1;34m(n_rows)\u001b[0m\n\u001b[0;32m     34\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mget_x_y\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_rows\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m     \u001b[0mdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdf_y\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_rows\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m     \u001b[1;31m#fill_nas(0, df)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-79-65ce3e0aac27>\u001b[0m in \u001b[0;36mget_data\u001b[1;34m(n_rows)\u001b[0m\n\u001b[0;32m     19\u001b[0m     \u001b[0mdf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdrop_dissent\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 21\u001b[1;33m     \u001b[0mdf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdummify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     22\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m     \u001b[1;31m# Extras -- for analysis\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-81-a5766d391670>\u001b[0m in \u001b[0;36mdummify\u001b[1;34m(df, max_uniq, sparse)\u001b[0m\n\u001b[0;32m     38\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 40\u001b[1;33m     \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mremove_for_now\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     41\u001b[0m     \u001b[0msum1\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/share/apps/pandas/0.17.1/intel/lib/python2.7/site-packages/pandas-0.17.1-py2.7-linux-x86_64.egg/pandas/core/generic.pyc\u001b[0m in \u001b[0;36mdrop\u001b[1;34m(self, labels, axis, level, inplace, errors)\u001b[0m\n\u001b[0;32m   1615\u001b[0m                 \u001b[0mnew_axis\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1616\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1617\u001b[1;33m                 \u001b[0mnew_axis\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1618\u001b[0m             \u001b[0mdropped\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreindex\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0maxis_name\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mnew_axis\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1619\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/share/apps/pandas/0.17.1/intel/lib/python2.7/site-packages/pandas-0.17.1-py2.7-linux-x86_64.egg/pandas/core/index.pyc\u001b[0m in \u001b[0;36mdrop\u001b[1;34m(self, labels, errors)\u001b[0m\n\u001b[0;32m   2801\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0many\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2802\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0merrors\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34m'ignore'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2803\u001b[1;33m                 \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'labels %s not contained in axis'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2804\u001b[0m             \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mindexer\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m~\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2805\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: labels ['Ads3' 'F1Ads3' 'F2Ads3' 'L1Ads3' 'L2Ads3' 'L3Ads3' 'L4Ads3' 'L5Ads3'\n 'Unnamed: 0.1' 'appel1' 'appel2' 'citevol' 'codej3' 'id' 'usc2sect'\n 'usc1sect' 'age2' 'distjudg' 'respond1' 'respond2' 'yearb' 'pred' 'csb'] not contained in axis"
     ]
    }
   ],
   "source": [
    "#############################################\n",
    "# Read data into X and y\n",
    "#############################################\n",
    "\n",
    "X, y = get_x_y()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(111538, 5015)\n",
      "(111538,)\n"
     ]
    }
   ],
   "source": [
    "print X.shape\n",
    "print y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  1.  11.  11. ...,   0.   0.   0.]\n",
      " [  1.  11.  11. ...,   0.   0.   0.]\n",
      " [  1.  11.  11. ...,   0.   0.   0.]\n",
      " ..., \n",
      " [  1.   5.   5. ...,   0.   0.   0.]\n",
      " [  1.   5.   5. ...,   0.   0.   0.]\n",
      " [  1.   5.   5. ...,   0.   0.   0.]]\n",
      "[1 1 1 1 1 1 1 1 1 1]\n"
     ]
    }
   ],
   "source": [
    "#sanity check\n",
    "print X[:10]\n",
    "print y[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#############################################\n",
    "# Split into training and test set\n",
    "#############################################\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.nbytes/1024/1024/1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 111538 entries, 0 to 111537\n",
      "Columns: 5015 entries, ElecYear_AndPrior to totalcites_nan\n",
      "dtypes: float64(5014), int64(1)\n",
      "memory usage: 593.1 MB\n"
     ]
    }
   ],
   "source": [
    "df_x.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(78076, 5015)\n",
      "(78076,)\n",
      "(33462, 5015)\n",
      "(33462,)\n"
     ]
    }
   ],
   "source": [
    "print X_train.shape\n",
    "print y_train.shape\n",
    "print X_test.shape\n",
    "print y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# #############################################\n",
    "# # Standard scale\n",
    "# #############################################\n",
    "\n",
    "# scaler = StandardScaler()\n",
    "# scaler.fit(X_train)\n",
    "\n",
    "# X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 12 candidates, totalling 36 fits\n",
      "{'n_estimators': 10, 'max_depth': 15}\n",
      "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
      "            max_depth=15, max_features='auto', max_leaf_nodes=None,\n",
      "            min_samples_leaf=1, min_samples_split=2,\n",
      "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
      "            oob_score=False, random_state=42, verbose=0, warm_start=False)\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.56      0.02      0.03      1370\n",
      "          1       0.96      1.00      0.98     32092\n",
      "\n",
      "avg / total       0.94      0.96      0.94     33462\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:  7.4min finished\n"
     ]
    }
   ],
   "source": [
    "#############################################\n",
    "# [OPTIONAL]\n",
    "# Random Forest Grid Search\n",
    "#############################################\n",
    "\n",
    "paramgrid = {'n_estimators': [10, 50, 100], 'max_depth': [1, 5, 10, 15]}\n",
    "\n",
    "rf_clf = RandomForestClassifier(random_state=42)\n",
    "\n",
    "grid_search(X_train, y_train, rf_clf, paramgrid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.71      0.00      0.01      1370\n",
      "          1       0.96      1.00      0.98     32092\n",
      "\n",
      "avg / total       0.95      0.96      0.94     33462\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#############################################\n",
    "# Random Forest\n",
    "#############################################\n",
    "\n",
    "# Replace labels (in case SVM was run)\n",
    "# y_train[y_train == 0.] = -1.\n",
    "# y_test[y_test == 0.] = -1.\n",
    "\n",
    "rf_clf = RandomForestClassifier(random_state=42, \n",
    "                                n_estimators=100, \n",
    "                                max_depth=15, \n",
    "#                                 class_weight={1.0: 1, -1.0: 150})\n",
    "                                )\n",
    "\n",
    "rf_clf.fit(X_train, y_train)\n",
    "\n",
    "y_pred = rf_clf.predict(X_test)\n",
    "\n",
    "print_report(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['unan', '0.184609815311']\n",
      "['unanimous', '0.153336130094']\n",
      "['sat_together_count', '0.0103510902689']\n",
      "['Wopinionlenght', '0.00792813593988']\n",
      "['lengthopin', '0.00743753616955']\n",
      "['Wlengthopin', '0.00718287353731']\n",
      "['Wtotalcites', '0.00512062512499']\n",
      "['pagelgth', '0.00509079287146']\n",
      "['propneg', '0.00500227552729']\n",
      "['negativecites', '0.00460555615382']\n"
     ]
    }
   ],
   "source": [
    "#############################################\n",
    "# [OPTIONAL]\n",
    "# Feature importance analysis\n",
    "#############################################\n",
    "\n",
    "top_n = get_top_n_feats(10,rf_clf.feature_importances_, get_columns(df_x))\n",
    "\n",
    "for t in top_n:\n",
    "    print t\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  49 tasks       | elapsed:    4.1s\n",
      "[Parallel(n_jobs=1)]: Done 199 tasks       | elapsed:   15.5s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 80 candidates, totalling 240 fits\n",
      "{'kernel': 'poly', 'max_iter': 1000, 'coef0': 1000.0, 'degree': 1, 'class_weight': {1.0: 1, -1.0: 150}}\n",
      "SVC(C=1.0, cache_size=200, class_weight={1.0: 1, -1.0: 150}, coef0=1000.0,\n",
      "  decision_function_shape=None, degree=1, gamma='auto', kernel='poly',\n",
      "  max_iter=1000, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False)\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.00      0.00      0.00        11\n",
      "          1       0.98      1.00      0.99       489\n",
      "\n",
      "avg / total       0.96      0.98      0.97       500\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 240 out of 240 | elapsed:   18.5s finished\n"
     ]
    }
   ],
   "source": [
    "#############################################\n",
    "# [OPTIONAL]\n",
    "# SVM Grid Search\n",
    "#############################################\n",
    "\n",
    "paramgrid = {'kernel': ['rbf', 'poly', 'sigmoid', 'linear'], \n",
    "             'degree': [1, 3, 5, 7, 9], \n",
    "             'coef0': [1e-3, 1e-1, 1e1, 1e3], \n",
    "             'max_iter': [1000], \n",
    "             'class_weight': [{1.0: 1, -1.0: 150}]}\n",
    "\n",
    "svm_clf = SVC()\n",
    "\n",
    "grid_search(X_train, y_train, svm_clf, paramgrid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.00      0.00      0.00        11\n",
      "          1       0.98      1.00      0.99       489\n",
      "\n",
      "avg / total       0.96      0.98      0.97       500\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#############################################\n",
    "# SVM\n",
    "#############################################\n",
    "\n",
    "# Replace labels\n",
    "# y_train[y_train == 0.] = -1.\n",
    "# y_test[y_test == 0.] = -1.\n",
    "\n",
    "svm_clf = SVC(kernel='rbf', max_iter=1000, coef0=1e-3, degree=2, class_weight={1.0: 1, -1.0: 150})\n",
    "\n",
    "svm_clf.fit(X_train, y_train)\n",
    "\n",
    "y_pred = svm_clf.predict(X_test)\n",
    "\n",
    "print_report(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1, -1])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tdf = pd.get_dummies(pd.read_csv('final_outs.csv', low_memory=False))\n",
    "\n",
    "# pd.unique(tdf.ix[:,1].values)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}