votelist3.0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from copy import deepcopy\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.set_option('max_rows',2000)\n",
    "pd.set_option('max_columns',2000)\n",
    "np.set_printoptions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#df = pd.read_stata('/scratch/sv1239/projects/mlcs/raw/Votelevel_stuffjan2013.dta')\n",
    "df = pd.read_stata('Votelevel_stuffjan2013.dta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(56403, 868)"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# columns to be removed, casenum or codej1... or j2vote... or j2maj... will be removed later\n",
    "del_cols = ['fileid','cite','vol','beginpg','endopin','endpage','docnum','priorpub','_merge','year',\n",
    "            'circuit','pseatno','decision_date','aatty_first_name','aatty_last_name','afirm_name',\n",
    "            'ratty_first_name','ratty_last_name','rname_of_first_listed_amicus_gro','rfirm_namew','decisiondatenew2',\n",
    "           'j1name','j2name','j3name','quartertoelect','pname','seatno','success','lsuc','ls1','ls2','ls3','lp',\n",
    "            'lp2','lp3','sseatno','congress','congreso','afirst_listed_amicus_group','yearquarter','name','Name','State','j',\n",
    "            'codej4','j4vote1','j4vote2','j4maj1','j4maj2','codej5','j5vote1','j5vote2','j5maj1','j5maj2',\n",
    "            'codej6','j6vote1','j6vote2','j6maj1','j6maj2','codej7','j7vote1','j7vote2','j7maj1','j7maj2',\n",
    "            'codej8','j8vote1','j8vote2','j8maj1','j8maj2','codej9','j9vote1','j9vote2','j9maj1','j9maj2',\n",
    "            'codej10','j10vote1','j10vote2','j10maj1','j10maj2','codej11','j11vote1','j11vote2','j11maj1','j11maj2',\n",
    "            'codej12','j12vote1','j12vote2','j12maj1','j12maj2','codej13','j13vote1','j13vote2','j13maj1','j13maj2',\n",
    "            'codej14','j14vote1','j14vote2','j14maj1','j14maj2','codej15','j15vote1','j15vote2','j15maj1','j15maj2','j16maj1','j16vote1']\n",
    "#\n",
    "# to_dummies = ['month','day','method','state','district','origin','source','distjudg',\n",
    "#               'applfrom','adminrev','opinstat','treat','classact','crossapp','counsel1','counsel2','sanction',\n",
    "#               'initiate','numappel','appnatpr','appnatpr','appbus','appnonp','appfed','appsubst','appstate',\n",
    "#               'appfiduc','ap_stid','genapel1','bank_ap1','genapel2','bank_ap2','appel1','appel2',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.drop(labels=del_cols,axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(56403, 762)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18585\n"
     ]
    }
   ],
   "source": [
    "## I use case id to find the record of each vote of the same case, caseList save each unique case id for one time. \n",
    "## There are 18000+ unique cases.\n",
    "\n",
    "caseList=[]\n",
    "for i in range(df.shape[0]):\n",
    "    if pd.notnull(df.ix[i][\"casenum\"]) & (df.ix[i][\"casenum\"] not in caseList):\n",
    "        caseList.append(df.ix[i][\"casenum\"])\n",
    "print len(caseList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60, 868)\n"
     ]
    }
   ],
   "source": [
    "## on my computer, about 0.5 second per case\n",
    "## there will be 6 rows for each case. codej1 correspond to primary judge \n",
    "newframe=pd.DataFrame()                ##  the rearrange of the original data\n",
    "output=[]                           ##   the corresponding alignment of judge 1 and judge 2, yes =1, no = -1\n",
    "for case in caseList[:10]:    \n",
    "    subtest=df[df.casenum==case].reset_index(drop=True)  ## 'subtest' only take the records that have a specific case id\n",
    "    num=subtest.shape[0]                                 ## num will be 3, because usally there are 3 records for each case \n",
    "    for j in range(num):\n",
    "        if pd.notnull(subtest.ix[j].codej1):\n",
    "            j1=subtest.ix[j].codej1\n",
    "        if pd.notnull(subtest.ix[j].codej2):\n",
    "            j2=subtest.ix[j].codej2\n",
    "        if pd.notnull(subtest.ix[j].codej3):\n",
    "            j3=subtest.ix[j].codej3\n",
    "    if subtest.ix[0].majvotes == 3:\n",
    "        j1j2=1\n",
    "        j1j3=1\n",
    "        j2j3=1\n",
    "    else:\n",
    "        if subtest.ix[0].direct1==subtest.ix[0].j2vote1:\n",
    "            j1j2=1\n",
    "        else:\n",
    "            j1j2=-1                \n",
    "        if subtest.ix[0].direct1==subtest.ix[0].j3vote1:\n",
    "            j1j3=1\n",
    "        else:\n",
    "            j1j3=-1\n",
    "        if subtest.ix[0].j2vote1==subtest.ix[0].j3vote1:\n",
    "            j2j3=1\n",
    "        else:\n",
    "            j2j3=-1\n",
    "    for j in range(num):\n",
    "        copytest=deepcopy(subtest.ix[j])\n",
    "        if copytest.ids==j1:\n",
    "            copytest.codej1=j1\n",
    "            newframe=newframe.append(copytest)\n",
    "            output.append(j1j2)\n",
    "            copytest.codej2=j3\n",
    "            copytest.codej3=j2\n",
    "            newframe=newframe.append(copytest)\n",
    "            output.append(j1j3)\n",
    "        if copytest.ids==j2:\n",
    "            copytest.codej1=j2\n",
    "            copytest.codej2=j1\n",
    "            newframe=newframe.append(copytest)\n",
    "            output.append(j1j2)\n",
    "            copytest.codej2=j3\n",
    "            copytest.codej3=j1\n",
    "            newframe=newframe.append(copytest)\n",
    "            output.append(j2j3)\n",
    "        if copytest.ids==j3:\n",
    "            copytest.codej1=j3\n",
    "            copytest.codej3=j1\n",
    "            newframe=newframe.append(copytest)\n",
    "            output.append(j2j3)\n",
    "            copytest.codej2=j1\n",
    "            copytest.codej3=j2\n",
    "            newframe=newframe.append(copytest)\n",
    "            output.append(j1j3)\n",
    "            \n",
    "newframe=newframe.reset_index()              ## need to reset the index, otherwise will all be 0\n",
    "newframe=newframe.drop('index',1)  \n",
    "\n",
    "assert newframe.shape[0]==len(output)\n",
    "print newframe.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## remove the columns that we are done with\n",
    "newframe.drop(labels=['casenum','j2vote1','j2vote2','j2maj1','direct1',\n",
    "                      'j2maj2','j3vote1','j3vote2','j3maj1','j3maj2','majvotes','ids'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "new_cols=newframe.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "new_cols=new_cols.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "keep_cols=['j1score','j2score','j3score','popularpct','electoralpct','closerd','fartherd','dAds3','dF2Ads3',\n",
    "           'dF1Ads3','dL1Ads3','dL2Ads3','dL3Ads3','dL4Ads3','dL5Ads3','logAds3','logL1Ads3','logL2Ads3','logF1Ads3',\n",
    "          'logF2Ads3','decade2','propneg','likely_elev2','score','d12','d13','d23']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for col in keep_cols:\n",
    "    if col in new_cols:\n",
    "        new_cols.remove(col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for i in new_cols:\n",
    "    #alist=pd.unique(df[i])\n",
    "    alist=pd.unique(newframe[i])\n",
    "    if len(alist)==3 or len(alist)==2: #       already check that 3 or 2 unique values won't have 20 as value, so use 20 to replace nan\n",
    "        #df[i].fillna(value=20,inplace=True)\n",
    "        newframe[i].fillna(value=20,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "do_dummies=[]\n",
    "for i in new_cols:\n",
    "    if len(pd.unique(df[i]))>3:\n",
    "        do_dummies.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "newframe=pd.get_dummies(newframe,columns=do_dummies,dummy_na=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "newframe=newframe.fillna(value=0)         ##  havn't checked the keep_cols yet, but most of them looks like don't have 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}