tools_data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from numpy import genfromtxt\n",
    "from numpy import mean\n",
    "from collections import Counter\n",
    "from collections import defaultdict\n",
    "import random\n",
    "import numpy\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "dataFiles = ['basic_event_frequency_count.csv', \n",
    "             'basic_event_frequency_count_with_session_min_size_500.csv',\n",
    "             'basic_event_tfidf.csv',\n",
    "             'basic_event_tfidf_with_session_min_size_500.csv',\n",
    "             'complex_event_frequency_count.csv',\n",
    "             'complex_event_frequency_count_with_session_min_size_500.csv',\n",
    "             'complex_event_tfidf.csv',\n",
    "             'complex_event_tfidf_with_session_min_size_500.csv',\n",
    "             'extra_command_event_frequency_count_min_size_500.csv',\n",
    "             'extra_command_event_tfidf_min_size_500.csv',\n",
    "             'extra_command_event_tfidf.csv',\n",
    "             'extra_command_event_frequency_count.csv']\n",
    "\n",
    "def readData(fileName):\n",
    "    # read in data\n",
    "    testData = genfromtxt(fileName, delimiter=',')\n",
    "\n",
    "    # extract features from data\n",
    "    x = [item[2:] for item in testData[1:]]\n",
    "\n",
    "    # extract programmer id from data\n",
    "    y = [int(item[0]) for item in testData[1:]]\n",
    "    return (x, y)\n",
    "\n",
    "def numberOfProgrammersWithXSessions(y):\n",
    "    # prints how many programmers have at least some value sessions\n",
    "    countOfProgId = Counter(y)\n",
    "    for i in range(1,31):\n",
    "        occurance = 0\n",
    "        for progId in countOfProgId.keys():\n",
    "            if(countOfProgId[progId] > i):\n",
    "                occurance += 1\n",
    "        print(str(i) + \" \" + str(occurance))\n",
    "\n",
    "def removeProgrammersWithLessThanThreshholdSessions(x, y, threshhold):        \n",
    "    # remove programmers from data with less than x sessions\n",
    "    countOfProgId = Counter(y)\n",
    "    progIdToKeep = [key for key in countOfProgId.keys() if countOfProgId[key] > threshhold]\n",
    "    newX = []\n",
    "    newY = []\n",
    "    for i in range(len(y)):\n",
    "        if y[i] in progIdToKeep:\n",
    "            newX.append(x[i])\n",
    "            newY.append(y[i])\n",
    "    x = newX\n",
    "    y = newY\n",
    "    return (x, y)\n",
    "\n",
    "def randomlySelectXSessionsForEachProgrammer(x, y, threshhold):   \n",
    "    # randomly select threshold sessions for each programmers\n",
    "    newX = []\n",
    "    newY = []\n",
    "    # make a dict with key progId and value a list of all session of that programmer \n",
    "    prodId = set(y)\n",
    "    dataGroupedByProg = defaultdict(list)\n",
    "    for i in range(len(x)):\n",
    "        dataGroupedByProg[y[i]].append(x[i]) \n",
    "        \n",
    "    #sample threshold from that list of list\n",
    "    for key in dataGroupedByProg.keys():\n",
    "        #random.shuffle(dataGroupedByProg[key])\n",
    "        sampledSessions = random.sample(dataGroupedByProg[key], threshhold)\n",
    "        #sampledSessions = dataGroupedByProg[key][:threshhold]\n",
    "        for i in range(len(sampledSessions)):\n",
    "            newX.append(sampledSessions[i])\n",
    "            newY.append(key)\n",
    "            \n",
    "    return (newX, newY)\n",
    "\n",
    "def processLinearSVCCrossVal(dataFiles):\n",
    "    for fileName in dataFiles:\n",
    "        data = readData(fileName)\n",
    "        data = removeProgrammersWithLessThanThreshholdSessions(data[0], data[1], 10)\n",
    "        (x, y) = randomlySelectXSessionsForEachProgrammer(data[0], data[1], 10)\n",
    "        xTrain, xTest, yTrain, yTest = train_test_split(x, y, test_size=0.2, random_state=45)\n",
    "        for i in [0.001, 0.0005,0.0004, 0.0003, 0.0002, 0.0001, 0.00009, 0.00008, 0.00005,  0.00001, 0.000001]:\n",
    "            clf = LinearSVC(C=i)\n",
    "            print(str(i) + \" \" + fileName)\n",
    "            print(mean(cross_val_score(clf, xTrain, yTrain, cv=5)))\n",
    "            print(\"\\n\")\n",
    "\n",
    "def processKNNCrossVal(dataFiles):            \n",
    "    for fileName in dataFiles:\n",
    "        data = readData(fileName)\n",
    "        data = removeProgrammersWithLessThanThreshholdSessions(data[0], data[1], 10)\n",
    "        (x, y) = randomlySelectXSessionsForEachProgrammer(data[0], data[1], 10)\n",
    "        xTrain, xTest, yTrain, yTest = train_test_split(x, y, test_size=0.2, random_state=45)\n",
    "        results = []\n",
    "        for i in range(1, 9):\n",
    "            results2 = []\n",
    "            for j in range (10):\n",
    "                nbrs = KNeighborsClassifier(n_neighbors = i)\n",
    "                nbrs.fit(x, y)\n",
    "                results2.append(mean(cross_val_score(nbrs, xTrain, yTrain, cv = 5)))\n",
    "            results.append(mean(results2))\n",
    "        print(fileName)    \n",
    "        print(results)  \n",
    "        \n",
    "def experimentOne():\n",
    "    dataFiles = ['basic_event_frequency_count_with_session_min_size_500.csv',\n",
    "             'basic_event_tfidf_with_session_min_size_500.csv',\n",
    "             'complex_event_frequency_count_with_session_min_size_500.csv',\n",
    "             'complex_event_tfidf_with_session_min_size_500.csv',\n",
    "             'extra_command_event_frequency_count_min_size_500.csv',\n",
    "             'extra_command_event_tfidf_min_size_500.csv']\n",
    "    for file in dataFiles:\n",
    "        data = readData(file)\n",
    "        data = removeProgrammersWithLessThanThreshholdSessions(data[0], data[1], 10)\n",
    "        (x, y) = randomlySelectXSessionsForEachProgrammer(data[0], data[1], 10)\n",
    "        xTrain1, xTest1, yTrain1, yTest1 = train_test_split(x, y, test_size=0.2, random_state = 42)\n",
    "        nbrs = KNeighborsClassifier(n_neighbors = 1).fit(xTrain1,yTrain1)\n",
    "        clf = LinearSVC(C=0.0001).fit(xTrain1,yTrain1)\n",
    "        clf2 = SVC(kernel='linear', C=0.0001, decision_function_shape = 'ovr', random_state=42)\n",
    "        clf2.fit(xTrain1, yTrain1)\n",
    "        print(file)\n",
    "        print(\"knn: \" + str(nbrs.score(xTest1, yTest1)))\n",
    "        print(\"Linear SVC: \" + str(clf.score(xTest1, yTest1)))\n",
    "        print(\"SVC: \" + str(clf2.score(xTest1, yTest1)))\n",
    "        print(\"\\n\")\n",
    "        \n",
    "def experimentTwo():\n",
    "    dataFiles = ['extra_command_event_frequency_count.csv',\n",
    "                 'extra_command_event_frequency_count_min_size_500.csv',\n",
    "                 'extra_command_event_frequency_count_min_size_1000.csv',\n",
    "                 'extra_command_event_frequency_count_min_size_1500.csv',\n",
    "                 'extra_command_event_frequency_count_min_size_2000.csv']\n",
    "\n",
    "    for fileName in dataFiles:\n",
    "        data = readData(fileName)\n",
    "        data = removeProgrammersWithLessThanThreshholdSessions(data[0], data[1], 10)\n",
    "        (x, y) = randomlySelectXSessionsForEachProgrammer(data[0], data[1], 10)\n",
    "        xTrain, xTest, yTrain, yTest = train_test_split(x, y, test_size=0.2, random_state = 42)\n",
    "        clf = LinearSVC(C=0.0001).fit(xTrain,yTrain)\n",
    "        print(str(len(xTrain)) + \" \" + str(len(xTest)))\n",
    "        print(fileName + \"\\n\" + str(clf.score(xTest, yTest)))\n",
    "    \n",
    "    \n",
    "def experimentThree():\n",
    "    for j in [5, 10, 15, 20, 25]:  \n",
    "        data = readData(dataFiles[8])\n",
    "        data = removeProgrammersWithLessThanThreshholdSessions(data[0], data[1], j)\n",
    "        (x, y) = randomlySelectXSessionsForEachProgrammer(data[0], data[1], j)\n",
    "        xTrain, xTest, yTrain, yTest = train_test_split(x, y, test_size=0.2, random_state = 42)\n",
    "        clf = LinearSVC(C=0.0001).fit(xTrain,yTrain)\n",
    "        print(str(j) + \" \" + str(clf.score(xTest, yTest)) + \" \" + str(len(xTest)) + \" \" + str(len(xTrain)))\n",
    "\n",
    "random.seed(42)\n",
    "experimentOne()\n",
    "experimentTwo()\n",
    "print()\n",
    "experimentThree()\n",
    "#processLinearSVCCrossVal(dataFiles)\n",
    "#processKNNCrossVal(dataFiles)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}