CW Task 2 v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pixiedust database opened successfully\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <div style=\"margin:10px\">\n",
       "            <a href=\"https://github.com/ibm-watson-data-lab/pixiedust\" target=\"_new\">\n",
       "                <img src=\"https://github.com/ibm-watson-data-lab/pixiedust/raw/master/docs/_static/pd_icon32.png\" style=\"float:left;margin-right:10px\"/>\n",
       "            </a>\n",
       "            <span>Pixiedust version 1.1.9</span>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.ml import *\n",
    "from pyspark.ml.classification import *\n",
    "from pyspark.ml.feature import *\n",
    "from pyspark.ml.param import *\n",
    "from pyspark.ml.tuning import *\n",
    "from pyspark.ml.evaluation import *\n",
    "from pyspark.sql import *\n",
    "import pixiedust\n",
    "from pyspark.sql.types import Row\n",
    "from pyspark.sql import SQLContext\n",
    "sqlContext = SQLContext(sc)\n",
    "spark = SparkSession.builder.getOrCreate()\n",
    "import pandas as pd\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Read in data, create schema for dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sc = spark.sparkContext # this is the Driver Program\n",
    "\n",
    "input_file_template=\"hdfs://saltdean/data/reviews/Reviews.csv\"\n",
    "df = sqlContext.read. \\\n",
    "  format(\"com.databricks.spark.csv\"). \\\n",
    "  option(\"header\",\"true\"). \\\n",
    "  load(input_file_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Id', 'string'),\n",
       " ('ProductId', 'string'),\n",
       " ('UserId', 'string'),\n",
       " ('ProfileName', 'string'),\n",
       " ('HelpfulnessNumerator', 'string'),\n",
       " ('HelpfulnessDenominator', 'string'),\n",
       " ('Score', 'string'),\n",
       " ('Time', 'string'),\n",
       " ('Summary', 'string'),\n",
       " ('Text', 'string')]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Drop irrelevant columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# drop all but the most important (consider adding back later)\n",
    "df = df.drop(df.Id).drop(df.ProductId).drop(df.ProfileName).drop(df.Summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#df = df.sample(False, 0.25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Inspect the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Show Helpfulness measure: numerator and denominator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+----------------------+\n",
      "|HelpfulnessNumerator|HelpfulnessDenominator|\n",
      "+--------------------+----------------------+\n",
      "|                   1|                     1|\n",
      "|                   0|                     0|\n",
      "|                   1|                     1|\n",
      "|                   3|                     3|\n",
      "|                   0|                     0|\n",
      "|                   0|                     0|\n",
      "|                   0|                     0|\n",
      "|                   0|                     0|\n",
      "|                   1|                     1|\n",
      "|                   0|                     0|\n",
      "+--------------------+----------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import functions as F\n",
    "df.select(\"HelpfulnessNumerator\", \"HelpfulnessDenominator\").show(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Inspect rows that have no ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+--------------------+----------------------+-----+----------+--------------------+\n",
      "|        UserId|HelpfulnessNumerator|HelpfulnessDenominator|Score|      Time|                Text|\n",
      "+--------------+--------------------+----------------------+-----+----------+--------------------+\n",
      "|A1D87F6ZCVE5NK|                   0|                     0|    1|1346976000|\"Product arrived ...|\n",
      "|A1UQRSCLF8GW1T|                   0|                     0|    5|1350777600|Great taffy at a ...|\n",
      "| ADT0SRK1MGOEU|                   0|                     0|    4|1342051200|I got a wild hair...|\n",
      "|A1SP2KVKFXXRU1|                   0|                     0|    5|1340150400|This saltwater ta...|\n",
      "|A3JRGQVEQN31IQ|                   0|                     0|    5|1336003200|This taffy is so ...|\n",
      "|A21BT40VZCCYT4|                   0|                     0|    5|1351209600|This is a very he...|\n",
      "|A3KLWF6WQ5BNYO|                   0|                     0|    2|1348099200|I love eating the...|\n",
      "| AFKW14U97Z6QO|                   0|                     0|    5|1345075200|I am very satisfi...|\n",
      "|A2A9X58G2GTBLP|                   0|                     0|    5|1324598400|Twizzlers, Strawb...|\n",
      "|A3IV7CL2C13K2U|                   0|                     0|    5|1318032000|Candy was deliver...|\n",
      "+--------------+--------------------+----------------------+-----+----------+--------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# look at rows that have a numerator and denominator of zero\n",
    "df.filter((df['HelpfulnessNumerator'] == 0) & (df['HelpfulnessDenominator'] == 0)).show(10)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Feature generation, data wrangling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Remove rows with no reviews and show result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# remove rows which have no reviews\n",
    "df = df.filter((df['HelpfulnessNumerator'] != 0) & (df['HelpfulnessDenominator'] != 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Remove reviews with less than 4 helpfulness votes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = df.filter(df['HelpfulnessDenominator'] >= 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Create new ratio variable: Helpfullness numerator / helpfulness denominator..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_new = df.withColumn('target', df.HelpfulnessNumerator / df.HelpfulnessDenominator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+--------------------+----------------------+-----+----------+--------------------+------+\n",
      "|        UserId|HelpfulnessNumerator|HelpfulnessDenominator|Score|      Time|                Text|target|\n",
      "+--------------+--------------------+----------------------+-----+----------+--------------------+------+\n",
      "|A2725IB4YY9JEB|                   4|                     4|    5|1282867200|One of my boys ne...|   1.0|\n",
      "|A2MUGFV2TDQ47K|                   4|                     5|    5|1268352000|The Strawberry Tw...|   0.8|\n",
      "|A1CZX3CP8IKQIJ|                   4|                     5|    5|1262044800|My daughter loves...|   0.8|\n",
      "| AOVROBZ8BNTP7|                  19|                    19|    4|1163376000|McCann's Instant ...|   1.0|\n",
      "|A3PMM0NFVEJGK9|                  13|                    13|    4|1166313600|This is a good in...|   1.0|\n",
      "+--------------+--------------------+----------------------+-----+----------+--------------------+------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_new.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Proportion of examples with a helpfulness ratio >= 0.75: 0.7087576374745418\n"
     ]
    }
   ],
   "source": [
    "df_ones = df_new.filter(df_new['target'] >= '0.75')\n",
    "print('Proportion of examples with a helpfulness ratio >= 0.75:', df_ones.count()/df_new.count())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### ... from that create a binary target variable: '1' if ratio is above 0.75"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+--------------------+----------------------+-----+----------+--------------------+------+-----------+\n",
      "|        UserId|HelpfulnessNumerator|HelpfulnessDenominator|Score|      Time|                Text|target|veryHelpful|\n",
      "+--------------+--------------------+----------------------+-----+----------+--------------------+------+-----------+\n",
      "|A2725IB4YY9JEB|                   4|                     4|    5|1282867200|One of my boys ne...|   1.0|          1|\n",
      "|A2MUGFV2TDQ47K|                   4|                     5|    5|1268352000|The Strawberry Tw...|   0.8|          1|\n",
      "|A1CZX3CP8IKQIJ|                   4|                     5|    5|1262044800|My daughter loves...|   0.8|          1|\n",
      "| AOVROBZ8BNTP7|                  19|                    19|    4|1163376000|McCann's Instant ...|   1.0|          1|\n",
      "|A3PMM0NFVEJGK9|                  13|                    13|    4|1166313600|This is a good in...|   1.0|          1|\n",
      "|A2EB6OGOWCRU5H|                   9|                     9|    5|1175212800|Instant oatmeal c...|   1.0|          1|\n",
      "|A3S5KJDA6ED2PS|                   4|                     4|    5|1243900800|\"Got a free packa...|   1.0|          1|\n",
      "|A1ZR8O62VSU4OK|                   2|                     4|    3|1318723200|Watch your prices...|   0.5|          0|\n",
      "|A2VOZX7YBT0D6D|                  15|                    15|    5|1325635200|\"I know the produ...|   1.0|          1|\n",
      "|A1FD9E5C06UB6B|                   5|                     5|    3|1301011200|While my dogs lik...|   1.0|          1|\n",
      "+--------------+--------------------+----------------------+-----+----------+--------------------+------+-----------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# create a new column 'veryHelpful' that will serve as the target variable \n",
    "from pyspark.sql.types import StringType\n",
    "from pyspark.sql.functions import udf\n",
    "veryHelpful_udf = udf(lambda target: '1' if target >= 0.75 else '0', StringType()) # EXPLAIN WHAT UDF DOES\n",
    "\n",
    "df = df_new.withColumn(\"veryHelpful\", veryHelpful_udf(df_new.target))\n",
    "df.show(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Bring relevant columns forward and change label to double (for compatibility with ML pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------------+-----+--------------+\n",
      "|label|                Text|Score|        UserId|\n",
      "+-----+--------------------+-----+--------------+\n",
      "|  1.0|One of my boys ne...|    5|A2725IB4YY9JEB|\n",
      "|  1.0|The Strawberry Tw...|    5|A2MUGFV2TDQ47K|\n",
      "|  1.0|My daughter loves...|    5|A1CZX3CP8IKQIJ|\n",
      "|  1.0|McCann's Instant ...|    4| AOVROBZ8BNTP7|\n",
      "|  1.0|This is a good in...|    4|A3PMM0NFVEJGK9|\n",
      "|  1.0|Instant oatmeal c...|    5|A2EB6OGOWCRU5H|\n",
      "|  1.0|\"Got a free packa...|    5|A3S5KJDA6ED2PS|\n",
      "|  0.0|Watch your prices...|    3|A1ZR8O62VSU4OK|\n",
      "|  1.0|\"I know the produ...|    5|A2VOZX7YBT0D6D|\n",
      "|  1.0|While my dogs lik...|    3|A1FD9E5C06UB6B|\n",
      "+-----+--------------------+-----+--------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = df.drop(\"HelpfulnessNumerator\").drop(\"HelpfulnessDenominator\").drop(\"target\").drop(\"Time\") # drop columns no longer needed\n",
    "df = df.select(\"veryHelpful\", \"Text\", \"Score\", \"UserId\")# change order of columns\n",
    "df = df.withColumnRenamed(\"veryHelpful\", \"label\") # rename binary target column to \"label\", in-keeping with pipeline terminology\n",
    "df = df.select(df.label.cast(\"double\"), df.Text, df.Score, df.UserId) # convert labels from strings to double (some ML algorithms dislike floats)\n",
    "df.show(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Create a feature that shows how many times a user has posted"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### First, remove duplicate reviews that will skew the 'count' variable:\n",
    "\n",
    "When a product has slight variations (e.g. different colour options), a user review is applied to all numerous product IDs.\n",
    "\n",
    "The duplicate examples are identical in every way except product ID. I.e. the text is exactly the same."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = df.dropDuplicates()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Now create the count column that will identify which users have genuinely posted more than once."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------+-----+--------------------+-----+-----------+\n",
      "|            UserId|label|                Text|Score|reviewCount|\n",
      "+------------------+-----+--------------------+-----+-----------+\n",
      "|#oc-R35YUP5T21KW2U|  0.0|Extremely disappo...|    1|          1|\n",
      "|    A10AZVYK32ZJSE|  1.0|Got this item bec...|    4|          1|\n",
      "|    A10Q1QIHVXKT8M|  1.0|I have two large ...|    5|          1|\n",
      "|    A11DAGJ4ZRNTDE|  1.0|\"I use this oil i...|    5|          1|\n",
      "|    A11G8FJESD56FN|  0.0|I had high expect...|    1|          2|\n",
      "|    A11G8FJESD56FN|  1.0|After reading the...|    1|          2|\n",
      "|    A11MCNZSTBW4UE|  0.0|I have 3 kids and...|    5|          1|\n",
      "|    A11N045YYLW4NU|  1.0|Very good product...|    5|          1|\n",
      "|    A11PM0C1979EZA|  1.0|This is the first...|    1|          1|\n",
      "|    A11Q181FHIVLF0|  0.0|I have been using...|    5|          1|\n",
      "+------------------+-----+--------------------+-----+-----------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "61327"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pyspark.sql import functions as F\n",
    "\n",
    "# aggregate the data\n",
    "df_count = df.groupby('UserId').agg(F.count(\"Score\").alias(\"reviewCount\"))\n",
    "\n",
    "# append back to df\n",
    "df = df.join(df_count, 'UserId')\n",
    "df.show(10)\n",
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# change reviewCount from 'bigint' to 'double'\n",
    "df = df.select(df.label, df.Text, df.Score, df.reviewCount.cast(\"double\"))\n",
    "\n",
    "# rename 'Text' column to 'reviewText'\n",
    "df = df.withColumnRenamed(\"Text\", \"reviewText\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Remove punctuation from 'reviewText'\n",
    "Source: https://mashimo.wordpress.com/tag/spark/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import regexp_replace, trim, col, lower\n",
    "\n",
    "def removePunctuation(column):\n",
    "  \"\"\"Removes punctuation, changes to lower case, and strips leading and trailing spaces.\n",
    "  Note:\n",
    "    Only spaces, letters, and numbers should be retained. Other characters \n",
    "    should should be eliminated (e.g. it's becomes its). Leading and \n",
    "    trailing spaces should be removed after punctuation is removed.\n",
    "  Args:\n",
    "    column (Column): A Column containing a sentence.\n",
    "  Returns:\n",
    "    Column: A Column named 'sentence' with clean-up operations applied.\n",
    "  \"\"\"\n",
    "  return trim(lower(regexp_replace(column, '([^\\s\\w_]|_)+', ''))).alias('reviewTextNoPunc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------------+-----+-----------+\n",
      "|label|    reviewTextNoPunc|Score|reviewCount|\n",
      "+-----+--------------------+-----+-----------+\n",
      "|  0.0|extremely disappo...|    1|        1.0|\n",
      "|  1.0|got this item bec...|    4|        1.0|\n",
      "|  1.0|i have two large ...|    5|        1.0|\n",
      "|  1.0|i use this oil in...|    5|        1.0|\n",
      "|  0.0|i had high expect...|    1|        2.0|\n",
      "|  1.0|after reading the...|    1|        2.0|\n",
      "|  0.0|i have 3 kids and...|    5|        1.0|\n",
      "|  1.0|very good product...|    5|        1.0|\n",
      "|  1.0|this is the first...|    1|        1.0|\n",
      "|  0.0|i have been using...|    5|        1.0|\n",
      "+-----+--------------------+-----+-----------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# remove punctuation\n",
    "df = df.select(df.label, removePunctuation(col('reviewText')), df.Score, df.reviewCount)\n",
    "df.show(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "We now have a data frame with the following input features:\n",
    "+ Text: the text contained within the review\n",
    "+ Score: the score for the product given by the reviewer\n",
    "+ Count: the number of times a user has posted: intuitively, the more a user posts, the more useful a review will be."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Create train and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total document count: 61327\n",
      "Training-set count: 55310\n",
      "Test-set count: 6017\n"
     ]
    }
   ],
   "source": [
    "#Create the training and testing set from the dataframe above\n",
    "#randomSplit - splits the Df into training/testing using the weights \n",
    "#you can try other combinations of weights\n",
    "train_set, test_set = df.randomSplit([0.9, 0.1], 123) # due to large number of examples,90/10 split chosen\n",
    "print (\"Total document count:\",df.count())\n",
    "print (\"Training-set count:\",train_set.count())\n",
    "print (\"Test-set count:\",test_set.count())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Set up the transformers and estimators / set up pipeline: Tokenizer, stopwords, hashing, IDF "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# packages\n",
    "from pyspark.ml.linalg import Vector\n",
    "from pyspark.ml import Pipeline, PipelineModel\n",
    "\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml.classification import GBTClassifier\n",
    "\n",
    "from pyspark.ml.feature import HashingTF,StopWordsRemover,IDF,Tokenizer\n",
    "from pyspark.ml.feature import IndexToString, StringIndexer, VectorIndexer, OneHotEncoder, VectorAssembler\n",
    "\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from pyspark.ml.tuning import ParamGridBuilder, CrossValidator\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "\n",
    "\n",
    "#Constructing a pipeline\n",
    "\n",
    "# String to index transformer e.g. Score is a string of 1-5, but need it as an index which can be one-hot encoded\n",
    "scoreIndexer = StringIndexer(inputCol = \"Score\", outputCol = \"indexedScore\")\n",
    "\n",
    "# One-hot encoder on the indexed feature(s)\n",
    "scoreEncoder = OneHotEncoder(inputCol = \"indexedScore\", outputCol = \"scoreVec\")\n",
    "\n",
    "#We split each sentence into words using Tokenizer. \n",
    "#Tokenizer only splits by white spaces\n",
    "tokenizer = Tokenizer().setInputCol(\"reviewTextNoPunc\").setOutputCol(\"words\")\n",
    "\n",
    "#Remove stopwords\n",
    "remover= StopWordsRemover().setInputCol(\"words\").setOutputCol(\"filtered\").setCaseSensitive(False)\n",
    "\n",
    "#For each sentence (bag of words),use HashingTF to hash the sentence into a feature vector. \n",
    "hashingTF = HashingTF().setInputCol(\"filtered\").setOutputCol(\"rawFeatures\") # .setNumFeatures(50000)\n",
    "\n",
    "#We use IDF to rescale the feature vectors; this generally improves performance when using text as features.\n",
    "idf = IDF().setInputCol(\"rawFeatures\").setOutputCol(\"idfFeature\").setMinDocFreq(0)\n",
    "\n",
    "# Assemble the vector of features\n",
    "assembler = VectorAssembler(inputCols = [\"scoreVec\", \"idfFeature\", \"reviewCount\"], outputCol = \"features\")\n",
    "\n",
    "#Our feature vectors could then be passed to a learning algorithm.\n",
    "# Create a Logistic regression model\n",
    "lr = LogisticRegression(labelCol = \"label\")\n",
    "\n",
    "# NOTE: there is an issue with the format of label. cf and rf don't like it when label is a float\n",
    "cf = DecisionTreeClassifier()\n",
    "rf = RandomForestClassifier(labelCol=\"label\", featuresCol=\"features\")\n",
    "nb = NaiveBayes()\n",
    "gbt = GBTClassifier(labelCol = \"label\", featuresCol = \"features\", maxIter=20)\n",
    "\n",
    "#Then basically we connect all the steps above to create one pipeline\n",
    "pipeline=Pipeline(stages=[scoreIndexer, scoreEncoder, tokenizer,remover,hashingTF,idf,assembler, lr])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Fit the  pipeline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.61 s, sys: 988 ms, total: 3.6 s\n",
      "Wall time: 1h 13min 25s\n"
     ]
    }
   ],
   "source": [
    "#Use the pipeline option to fit the training set and create a model\n",
    "\n",
    "# Jupyter offers a simpler way to take the time thanwe used in the coursework. \n",
    "\n",
    "\n",
    "# After we construct this ML pipeline,we can fit it to the training data\n",
    "# and obtain a trained pipeline model that can be used for prediction.\n",
    "%time model=pipeline.fit(train_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Evaluate the pipeline (without tuning hyperparameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+----------+-----+\n",
      "|    reviewTextNoPunc|         probability|prediction|label|\n",
      "+--------------------+--------------------+----------+-----+\n",
      "|and they arent so...|[4.53377539254321...|       1.0|  0.0|\n",
      "|dear amazoncombr ...|[0.23597536770555...|       1.0|  0.0|\n",
      "|i am laughing whi...|[1.0,1.5278552527...|       0.0|  0.0|\n",
      "|i grew up drinkin...|[0.26776949661259...|       1.0|  0.0|\n",
      "|i say the food ma...|[0.99987128496414...|       0.0|  0.0|\n",
      "|okay im a very ca...|[2.38675150682009...|       1.0|  0.0|\n",
      "|the miracle noodl...|[0.99565205274726...|       0.0|  0.0|\n",
      "|for those who hav...|[2.11567190293639...|       1.0|  1.0|\n",
      "|got two big bags ...|[0.00100552318697...|       1.0|  1.0|\n",
      "|i am a soda fan a...|[9.71925377304958...|       1.0|  1.0|\n",
      "|i am disgusted th...|[2.68567237316056...|       1.0|  1.0|\n",
      "|i bought the bund...|[9.94716747075336...|       1.0|  1.0|\n",
      "|i dont have cance...|[0.07634338426909...|       1.0|  1.0|\n",
      "|i drink strong bl...|[2.45731098898060...|       1.0|  1.0|\n",
      "|i found a bottle ...|[3.22942209015979...|       1.0|  1.0|\n",
      "|i had farro at a ...|[0.00207449652694...|       1.0|  1.0|\n",
      "|i had gastric byp...|[1.50005788430742...|       1.0|  1.0|\n",
      "|i love coffee and...|[1.06764581153731...|       1.0|  1.0|\n",
      "|i love this produ...|[0.86816673903800...|       0.0|  1.0|\n",
      "|i started using t...|[0.01041015371361...|       1.0|  1.0|\n",
      "+--------------------+--------------------+----------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# After we obtain a fitted pipeline model, we want to know how well it performs. \n",
    "\n",
    "test_predictions = model.transform(test_set) #transform() used to make predictions on the test set\n",
    "train_predictions = model.transform(train_set)\n",
    "\n",
    "#Show the predicted labels along with true labels and raw texts.\n",
    "test_predictions.select(\"reviewTextNoPunc\",\"probability\",\"prediction\",\"label\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Area under precision-recall curve - training: 0.9997293852434521\n",
      "Area under precision-recall curve - testing: 0.7873379023798541\n"
     ]
    }
   ],
   "source": [
    "# evaluate the model quantitatively.\n",
    "\n",
    "# for GBT (also works for dt, :\n",
    "#evaluator = MulticlassClassificationEvaluator().setMetricName(\"f1\") #  (supports \"f1\" (default), \"weightedPrecision\", \"weightedRecall\", \"accuracy\")\n",
    "\n",
    "# for LR:\n",
    "evaluator = BinaryClassificationEvaluator().setMetricName(\"areaUnderPR\") # supports \"areaUnderROC\" (default), \"areaUnderPR\"\n",
    "\n",
    "print (\"Area under precision-recall curve - training:\",evaluator.evaluate(train_predictions))\n",
    "print (\"Area under precision-recall curve - testing:\",evaluator.evaluate(test_predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Tune model with hyperparameters\n",
    "## Set up the grid search parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### print an explanation of the parameters available to LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "elasticNetParam: the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty. (default: 0.0)\n",
      "featuresCol: features column name. (default: features)\n",
      "fitIntercept: whether to fit an intercept term. (default: True)\n",
      "labelCol: label column name. (default: label, current: label)\n",
      "maxIter: max number of iterations (>= 0). (default: 100)\n",
      "predictionCol: prediction column name. (default: prediction)\n",
      "probabilityCol: Column name for predicted class conditional probabilities. Note: Not all models output well-calibrated probability estimates! These probabilities should be treated as confidences, not precise probabilities. (default: probability)\n",
      "rawPredictionCol: raw prediction (a.k.a. confidence) column name. (default: rawPrediction)\n",
      "regParam: regularization parameter (>= 0). (default: 0.0)\n",
      "standardization: whether to standardize the training features before fitting the model. (default: True)\n",
      "threshold: Threshold in binary classification prediction, in range [0, 1]. If threshold and thresholds are both set, they must match.e.g. if threshold is p, then thresholds must be equal to [1-p, p]. (default: 0.5)\n",
      "thresholds: Thresholds in multi-class classification to adjust the probability of predicting each class. Array must have length equal to the number of classes, with values >= 0. The class with largest value p/t is predicted, where p is the original probability of that class and t is the class' threshold. (undefined)\n",
      "tol: the convergence tolerance for iterative algorithms (>= 0). (default: 1e-06)\n",
      "weightCol: weight column name. If this is not set or empty, we treat all instance weights as 1.0. (undefined)\n"
     ]
    }
   ],
   "source": [
    "print(lr.explainParams())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "binary: If True, all non zero counts are set to 1. This is useful for discrete probabilistic models that model binary events rather than integer counts. Default False. (default: False)\n",
      "inputCol: input column name. (current: filtered)\n",
      "numFeatures: number of features. (default: 262144)\n",
      "outputCol: output column name. (default: HashingTF_4279af57f656e3e671b2__output, current: rawFeatures)\n"
     ]
    }
   ],
   "source": [
    "print(hashingTF.explainParams())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Set up the parameter grid to determine the impact of the regularisation parameter and the hashingTF vector size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "#We use a ParamGridBuilder to construct a grid of parameters to search over.\n",
    "\n",
    "#With 3 values for hashingTF.numFeatures and 3 values for idf,\n",
    "# this grid will have 3 x 3 = 9 parameter settings for CrossValidator to choose from. ############################\n",
    "lr_reg_values = list(np.logspace(-5, 5, 3))\n",
    "lr_elas_values = [0.1, 0.5, 0.9]\n",
    "\n",
    "# i.e. the feature preprocessing step\n",
    "#i.e. the machine learning parameter\n",
    "paramGrid = ParamGridBuilder()\\\n",
    "    .addGrid(hashingTF.numFeatures,[50, 500, 2500])\\\n",
    "    .addGrid(lr.regParam, lr_reg_values)\\\n",
    "    .build()\n",
    "\n",
    "    #   .addGrid(lr.elasticNetParam, lr_elas_values)\\ \n",
    "# .addGrid(hashingTF.numFeatures,[1000, 5000, 10000, 500000, 100000])\\\n",
    "      \n",
    "#.addGrid(idf.minDocFreq,[0,10,100])\\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{Param(parent='HashingTF_4279af57f656e3e671b2', name='numFeatures', doc='number of features.'): 50,\n",
       "  Param(parent='LogisticRegression_4b2da37ddbfb29079c71', name='regParam', doc='regularization parameter (>= 0).'): 1e-05},\n",
       " {Param(parent='HashingTF_4279af57f656e3e671b2', name='numFeatures', doc='number of features.'): 50,\n",
       "  Param(parent='LogisticRegression_4b2da37ddbfb29079c71', name='regParam', doc='regularization parameter (>= 0).'): 1.0},\n",
       " {Param(parent='HashingTF_4279af57f656e3e671b2', name='numFeatures', doc='number of features.'): 50,\n",
       "  Param(parent='LogisticRegression_4b2da37ddbfb29079c71', name='regParam', doc='regularization parameter (>= 0).'): 100000.0},\n",
       " {Param(parent='HashingTF_4279af57f656e3e671b2', name='numFeatures', doc='number of features.'): 500,\n",
       "  Param(parent='LogisticRegression_4b2da37ddbfb29079c71', name='regParam', doc='regularization parameter (>= 0).'): 1e-05},\n",
       " {Param(parent='HashingTF_4279af57f656e3e671b2', name='numFeatures', doc='number of features.'): 500,\n",
       "  Param(parent='LogisticRegression_4b2da37ddbfb29079c71', name='regParam', doc='regularization parameter (>= 0).'): 1.0},\n",
       " {Param(parent='HashingTF_4279af57f656e3e671b2', name='numFeatures', doc='number of features.'): 500,\n",
       "  Param(parent='LogisticRegression_4b2da37ddbfb29079c71', name='regParam', doc='regularization parameter (>= 0).'): 100000.0},\n",
       " {Param(parent='HashingTF_4279af57f656e3e671b2', name='numFeatures', doc='number of features.'): 2500,\n",
       "  Param(parent='LogisticRegression_4b2da37ddbfb29079c71', name='regParam', doc='regularization parameter (>= 0).'): 1e-05},\n",
       " {Param(parent='HashingTF_4279af57f656e3e671b2', name='numFeatures', doc='number of features.'): 2500,\n",
       "  Param(parent='LogisticRegression_4b2da37ddbfb29079c71', name='regParam', doc='regularization parameter (>= 0).'): 1.0},\n",
       " {Param(parent='HashingTF_4279af57f656e3e671b2', name='numFeatures', doc='number of features.'): 2500,\n",
       "  Param(parent='LogisticRegression_4b2da37ddbfb29079c71', name='regParam', doc='regularization parameter (>= 0).'): 100000.0}]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paramGrid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Evaluation using TrainValidationSplit() - note: faster but less reliable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "TrainValidationSplit() is less expensive but won't produce as relaible results on a small training set.\n",
    "\n",
    "A TrainValidationSplit requires an Estimator, a set of Estimator ParamMaps, and an Evaluator."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+----------+-----+\n",
      "|    reviewTextNoPunc|         probability|prediction|label|\n",
      "+--------------------+--------------------+----------+-----+\n",
      "|and they arent so...|[0.29251681635931...|       1.0|  0.0|\n",
      "|dear amazoncombr ...|[0.29369101918999...|       1.0|  0.0|\n",
      "|i am laughing whi...|[0.41717554831513...|       1.0|  0.0|\n",
      "|i grew up drinkin...|[0.43688558066758...|       1.0|  0.0|\n",
      "|i say the food ma...|[0.29966129702375...|       1.0|  0.0|\n",
      "|okay im a very ca...|[0.21874843935537...|       1.0|  0.0|\n",
      "|the miracle noodl...|[0.44291340913706...|       1.0|  0.0|\n",
      "|for those who hav...|[0.15779719800194...|       1.0|  1.0|\n",
      "|got two big bags ...|[0.24231556672479...|       1.0|  1.0|\n",
      "|i am a soda fan a...|[0.20824775775624...|       1.0|  1.0|\n",
      "|i am disgusted th...|[0.43759026082506...|       1.0|  1.0|\n",
      "|i bought the bund...|[0.23596344363638...|       1.0|  1.0|\n",
      "|i dont have cance...|[0.35024227884077...|       1.0|  1.0|\n",
      "|i drink strong bl...|[0.16411711894234...|       1.0|  1.0|\n",
      "|i found a bottle ...|[0.21991130499364...|       1.0|  1.0|\n",
      "|i had farro at a ...|[0.22773396345929...|       1.0|  1.0|\n",
      "|i had gastric byp...|[0.20203418593788...|       1.0|  1.0|\n",
      "|i love coffee and...|[0.21521311643067...|       1.0|  1.0|\n",
      "|i love this produ...|[0.15472538217734...|       1.0|  1.0|\n",
      "|i started using t...|[0.20600757176660...|       1.0|  1.0|\n",
      "+--------------------+--------------------+----------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tvs = TrainValidationSplit(estimator=pipeline,\n",
    "                           estimatorParamMaps=paramGrid,\n",
    "                           evaluator=BinaryClassificationEvaluator().setMetricName(\"areaUnderPR\"),\n",
    "                           # 80% of the data will be used for training, 20% for validation.\n",
    "                           trainRatio=0.8)\n",
    "\n",
    "tuned_model = tvs.fit(train_set) # fit the optimal model as determined via TVS gridsearch\n",
    "\n",
    "# Make predictions on test data. model is the model with combination of parameters\n",
    "# that performed best.\n",
    "#model.transform(test_set)\\\n",
    "   # .select(\"Text\", \"probability\", \"prediction\", \"label\")\\\n",
    "   # .show()\n",
    "\n",
    "#You can simply use the .transform() on the test set to make predictions on the test set\n",
    "test_predictions = tuned_model.transform(test_set)\n",
    "train_predictions = tuned_model.transform(train_set)\n",
    "\n",
    "test_predictions.select(\"reviewTextNoPunc\",\"probability\",\"prediction\",\"label\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.8966537369590957,\n",
       " 0.887640603120219,\n",
       " 0.8559673169693522,\n",
       " 0.8998961540883478,\n",
       " 0.8934771735388618,\n",
       " 0.8527657449323489,\n",
       " 0.8956012084491871,\n",
       " 0.9008419918917835,\n",
       " 0.8686924087892259]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = tuned_model.validationMetrics\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#grid_results = tuned_model.validationMetrics\n",
    "grid_reshape = np.reshape(results, (3,3)) ######### check this is the right way around...\n",
    "#grid_reshape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Heatmap of ML param grid search results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x7f55db953898>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAELCAYAAAASrNdlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XecVPX1//HXmd2lw9KWXpSiIFJEICKgCFYsKPYkxJ7E\nFkN+oNiiaBSjUZN8NbHEgvr1i7FFwAJRRAEboDTpKGWBpXek7ZzfHzMsu8uWUXZ35s68nz7m4dx7\nP/feM8ty+My5n/u55u6IiEgwhOIdgIiIxE5JW0QkQJS0RUQCRElbRCRAlLRFRAJESVtEJECUtEVE\nDpOZnWlmC8xskZndVsT2Fmb2oZnNMrOJZtYk37YrovstNLNflXquRBunXfXMxxIroCTUskuHeIeQ\n9Cbf2T/eIaSErJrpdrjHqHrcTTHnnB++eeKQ85lZCFgE9AdWA9OAy9x9Qb42/wbGuPsrZtYXuNrd\nf2VmdYDpQFfAgBlAV3ffWlwM6mmLSGqzUOyvovUAFrv7cnffB4wGBhZqcwwwEcDdJ+XbfgYwwd23\nuvsWYAJwZknhKmmLSGozi/1VtKbAynzL2dF1+c0ELoyczgYBNaK97ML7ripi3wKUtEUktR1+T7uo\nbF645DIM6GtmM4A+RJLz/hj3LSC95E8jIpLkQmnFbsrdtpLw9oMdYTPrGy1v5JcNtMi33IxIbTuP\nu6/hYE+7OnChu283s2ygb6F9Py4pXCVtEUltxZc9SMtsQVrmwXy8f9Vnk4poNg1oY2YtgTXAZcDl\nBU9h9YBNHhn5cTvwfHTTeOABM8skUvk4DRheUrgqj4hIajvM8oi75wI3EbmI+C0w2t3nm9kIMzsn\n2qwvsNDMFgANgAei+24G7icyguRLYET0gmSx1NMWkdRWQk87Vu7+AXB0oXX35Hv/JvBmMfu+CLwY\n67mUtEUktRV/gTEhKWmLSGorg552RVLSFpHUpp62iEiAlDDkLxEpaYtIalNPW0QkQEKqaYuIBId6\n2iIiAaLRIyIiAaKetohIgKinLSISIBryJyISICqPiIgEiMojIiIBop62iEiAqKctIhIg6mmLiASI\nkraISIBoyJ+ISICopi0iEiAqj4iIBIh62iIiwWFK2iIiwaGkLSISIKYn1ySv044/gkd+25eQGaPG\nz+XR16cV2N48qyZP/eF06mdWY9O2H7j64fdZs2lnnKINjt5H1eeOc44mZMYb01fxr0++L7C9UWYV\nHrrkWGpWySBk8NgHi5m8aAPpIWPEoGM4tmkmue6MHLuAad9vjtOnSGxffDaZvz/6Z8Ie5pzzBvHL\nK68tsH1tzhoeuPcOdmzfTtjD/ObGIfTs1QeAJYsX8peR97Fzxw5CaSH+9dJrZGRUisfHKBfqaScp\nM3j8xn4MGP46qzfuZMrff87Yz5ewKPtgkhh53Um8/N95jJ44nz6dmnH/1X249i8fxDHqxGcGd5/X\nnqv+NY112/bw+k0n8NG8dXy//uA/dtf3a8X7s3J47atsWmVV55mrunLqw5O5uEcz3GHg3z6jTvUM\nnr3qeC564os4fprEFA6HefzhB/jbP5+nflYW1/7qUvr07UfLI1rltRn13NP0O+0szr/wEpZ9v5Rh\nt1zP62MmkJuby/1/HM499z9MqzZt2bZtK+npGXH8NGUvaEk7WGNd4qj70Y1ZsmozK9ZtZ39umDc+\nWci5PdsUaNOuRT0+mbkCgMmzszmnZ+t4hBoonZplsnzjTlZv2c3+sPPerBz6H9OgQJuwO9WrRPoX\ntaqms3bbHgDaNKjBF0s2AbB55z62/bCfY5vWqtgPEADzvp1DsxYtadS4CenpGfQ//SwmT5pYoE0o\nFGLXzh0A7Ni+nfpZkT+Dr774jDZt29GqTVsAatXKDFySK42ZxfxKBEraMWpSrwbZ67fnLWdv2EGT\n+jUKtJn93XrO7x355R7Yqw01qmRQu0blCo0zaBpmViFny+685Zytu2lYq+DP7MkPlzLwuCZ8PPwk\nnrqiK38aMx+ABWu20++YLEIGTetUpUOzWjSqXaVC4w+CDevW0qBho7zlBg0asWH9ugJtrrruesa/\nN5ZBZ/fn1iE3MGTYnQCsXLEMgD/c/GuuGXwJr770fIXFXWHsR7wSQLmWR8ysHTAQaAo4sBoY4+7z\ny/O85aGof2TdvcDyHf/6lMdv6McvT+vA1DnZrN64g/254QqKMJiK+nvghZbP7tKYt2asYtSU5XRu\nnsnDl3bi3Men8ub0bFo3qM7rN/Vk9ZYf+GbZZnLDhfeWwr+nwCG/0B+Of48B557Ppb+4grlzZnHf\nH2/jlX+PITc3lzmzvuFfL79G5UqVueWGa2h3TAe6dvtZBUVf/hKlBx2rckvaZnYbcDkwGvgquroZ\n8H9mNtrdHypqv9xNSwhvWpq3HKrbmrS6bYpqWqFWbdhB8wYHv3o3q1+DNRsLXmTM2bSTy/80FoBq\nldM5v3dbdvywr0LjDJqcrbtpXLtq3nKjzCqsi5Y/DriwW1OufX4GALNWbqVyeoja1TLYsmsfD727\nMK/dq7/twbINuyom8ADJatiItTlr8pbXrcuhfv2sAm3GvfMWjz3xDADHduzM3j172bJlMw0aNKRL\n127UqpUJwAkn9mHhgvlK2nFUnuWRa4Du7v6Qu78SfT0E9IhuK1Ja3TZktDkj75UICRtg+qIcWjep\nTYsGNclID3HRyUcz7oulBdrUrXnwq/mwy3owasK3FR1m4MzJ3kqLetVoUrsKGWnGgM6NmDiv4Ff3\n1Vt2c2KbegC0yqpOpfQQW3bto3J6iCoZkV/hE9vUY3/YC1zAlIj2xxzLqpUryFmzmn379vLRhPfp\nffIpBdo0atyE6V99DsCy75eyb99eateuQ4+evVi6ZBF79uxh//79zPx6OkccmVzXakKhUMyvRFCe\n5ZEw0ARYXmh94+i2QAmHnSFPTmTsgxfmDflbuHITd/2yJzMW5fD+V99zUqfm3HdVb8LuTJmTze+f\nnFj6gVNc2OH+MfN57ppumMGb01bx3fqd3Hxqa+Zkb2PSgvU8/O5C7r+wA1f0bknYYfi/5wBQr0Yl\n/nX18eQ6rNu6m1tfmxPnT5OY0tLSGHLrnQy58Trcw5w9cBBHHNma555+gnbHHEuvPn258fdD+fOf\n7uG1V18iZCHuvPdBAGrWrMWlv/gV1w6+hFAoRM/eJ+UNBUwawepoY0XWu8riwGZnAk8Ai4GV0dUt\ngDbATe5e5Fi4qmc+pqJkOWvZpUO8Q0h6k+/sH+8QUkJWzfTDTrn1rxwdc87Z8OJlcU/x5dbTdvcP\nzOwoIuWQpkT+PcsGprl7bnmdV0TkxwhaTbtcR4+4exjQ3Q4ikrCUtEVEgiRYOVtJW0RSm3raIiIB\nkihD+WIVrGhFRMpYWcw9YmZnmtkCM1sUvbGw8PbmZjbRzL42s5lmdlZ0fYaZPW9ms83sGzM7ubR4\n1dMWkdR2mNURMwsRGd7cn8hUHdPM7B13X5Cv2V3Aa+7+tJm1B94DjgSuA9zdO5lZFvA+0K2k86mn\nLSIprQx62j2Axe6+3N33EZm6Y2ChNmHgwDwYtYFV0ffHAB8BuPt6YIuZKWmLiBSnDJJ2Uw7eQAiR\n+1GaFmozAhhsZiuBccDN0fWzgIFmlmZmRwLHA81LilflERFJaSXVqnevmsOeVXPztb2gr7tPKnyI\nInYtfJfl5cAL7v64mZ0AvAJ0AJ4H2gPTiEz5MRXYX1K8StoiktpKqGlXadaRKs065i1vmzZ6UhHN\nsolM0XFAMyK17fyuAc4AcPcvzKyKmdV39w3AH/JCMZtKZOqPYqk8IiIprQxm+ZsGtDGzlmZWCbgM\nGFOozXLgVIDohcjK7r7BzKqaWbXo+tOAfYUuYB5CPW0RSWmHe3ONu+ea2U3ABCId4efcfb6ZjSAy\n19I4YCjwrJkNIXJR8oro7g2A8WaWS+Ti5ODSzqekLSIprSzuiIzOWnp0oXX35Hs/H+hdxH7LgXY/\n5lxK2iKS2oJ1F7uStoikNs09IiISIEraIiIBErCcraQtIqktFApW1lbSFpGUpvKIiEiABCxnK2mL\nSGpTeUREJEDU0xYRCRDVtEVEAkTlERGRAFFPW0QkQAKWs5W0RSS1qactIhIgAcvZStoiktrU0xYR\nCZCA5WwlbRFJbRryJyISICqPHK5NhZ88L2VtzfKseIeQ9GpWTby/WlK0gOXsBEzaIiIVSD1tEZEA\nCVjOVtIWkdSmnraISIAELGcraYtIaguFQvEO4UdR0haRlKaetohIgAStph3T9wIzO8fMvjGzTWa2\nzcy2m9m28g5ORKS8mcX+SgSx9rT/CgwC5ri7l2M8IiIVKmg97ViT9kpgrhK2iCSbgOXsmJP2rcB7\nZvYJsOfASnd/rFyiEhGpIKGAZe1Yk/YDwA6gClCp/MIREalYyTrLXxN3P7ZcIxERiYOA5ezYRo8Q\nKY2cXq6RiIjEgZnF/EoEsfa0rweGmtkeYB9ggLt7rXKLTESkAiRILo5ZTEnb3WuWdyAiIvFgBCtr\nx3xHpJnVAdoSuRgJgLt/Wh5BiYhUlKSsaZvZtcCnwHhgRPT/95ZfWCIiFaMsatpmdqaZLTCzRWZ2\nWxHbm5vZRDP72sxmmtlZ0fXpZvaimc02s2/NbHhp8cZ6IfIWoDuw3N1PAY4DtsS4r4hIwkoLWcyv\nophZCHgCOAPoAFxuZu0KNbsLeM3duwKXA/+Irr8YqOTunYBuwG/MrEVJ8caatHe7++5ogJXdfQFw\ndIz7iogkrDKYe6QHsNjdl7v7PmA0MLBQmzBwYOBGbWBV9L0D1c0sDahG5ObFEud1irWmnW1mtYH/\nAP81s83A8hj3FRFJWGUwlK8pkak+DsgmksjzGwFMMLPfEUnOp0bXv0Ekwa8BqgJD3L3EKkaso0cu\niL6918w+BjKBD2LZV0QkkZWUs9ctmM76BTMOtr36mb7uPqnwIYrYtfA8TZcDL7j742Z2AvAKkVLK\nz4D9QCOgHjDZzD5092XFxVRq0o7Wa+a5ezsAd/+ktH1ERIKipLlHGrXvTqP23fOW5/3n6UlFNMsG\n8tehmwGrC7W5hkjNG3f/wswqm1l9Isn8A3cPA+vNbCqR2vayYuMt4bMQPUEYWFhacVxEJIjsR7yK\nMQ1oY2YtzawScBkwplCb5URLImbWHqji7huAFUC/6PrqwAnAgpLijbWmXQf41sy+AnYeWOnu58W4\nv4hIQipuVEis3D3XzG4CJhDpCD/n7vPNbAQwzd3HAUOBZ81sCJGLkldEd38SeMHM5kaXn3P3uZQg\n1qR994/9ICIiQVAWc4q4+wcUGlHn7vfkez8f6F3EfjuBS37MuWK9EKk6togkpaDNPRLrHZEnmNk0\nM9thZnvNLFfPiBSRZJCss/w9QaS4/jqRK5u/Ao4qr6BERCpKUs49AuDuS4A0d8919xeAM8svLBGR\nipGsPe1d0aEsM83sYSJ378Sc8EVEElVipOLYxZp4B0fb3kRkyF9z4MLyCkpEpKIc7oRRFS2WOyKP\nA1oD30aHrYwo96gS1GknHM0jfxhIKGSMGvMVj770cYHtzRvW5qm7L6F+nRps2rqLq//4Kms26Hpt\nafp3bsJDg7sRChkvf7yEv479tsD2pnWr8dQNvcisloGZMWL0N3w4azXpacZfrz2B41rVIxx2hr80\njanz18XpUyS2qZM/5eE/P0g47Fww6EKuvvbXBbbnrFnDXXfcxvbt2/FwmN8N+QO9+5zM6tWruODc\nARxxZCsAOnXqzJ1/vDcOn6D8JErZI1YlJm0z+yPwS2AG8LCZjXT3ZysksgRjZjw+7AIG3Pg0q9dv\nZcqoWxj7yVwWLV+f12bkLefy8rjpjP7ga/p0bc39Nw3g2ntHxzHqxGcGf7myO+c98CFrNu/i4z8N\n4N0ZK1m8+uA/dsMu6Mhbny/jhY8Wc1STWrxxW3863fI2V/ZrCw69bhtHvZqVeXN4f/re+V78PkyC\nCofDjHzgfp55/kWyshrwi0sv4pR+/TmyVeu8Ns88/U/OOGsAF19yGd8tXcqN11/H+xMmAtC8RQte\ne+PteIVf7gKWs0stj1wKdHH3y4nMp/3rUtonre4dmrNk5QZW5Gxmf26YNybM5NyTCz6gvt2RDflk\n+hIAJn+9lHNO6hCPUAPl+Nb1WZqznZUbdrI/13nr82WcfXzzAm3C7tSsmgFAZvVKrN60C4Cjm2Yy\nae4aADZu38PWnXs5rlXdCo0/CObOmU2Lli1p0qQpGRkZnHHW2Xw88aMCbUIhY+eOHQBs376NBg0a\n5m3zwlMfJZmQWcyvRFBa0t7t7rsA3H1jDO2TVpOsTLLXHpwxMXvdVppkFXyu8exFqzi/X0cABvY9\nlhpVK1O7ZtUKjTNomtStxqqNu/KWV23aReO61Qq0eejN2VzapxXfPjGIfw/rx62jvgJg7orNnN2t\nOSEzWmbVoMuRdWlat3qFxh8E69aupVGjxnnLDRs1ZN26gmWk315/E+PGvsPp/U/m5ht+y/A7D94E\nvXpVNpddPIhrrhzM1zOmV1jcFaUM5tOuUKXVtFub2YGJT6zQ8k+ee8TMrooOGzxE7raVhLcfnJo2\nVLM5abWaF9W0QhVV9yrcA7nj7+N4fNgF/PLs7kyd+R2r129lf264giIMpiLntCz0g73oxCP530lL\n+cf78+nWpj7P3NibE4aN5eWPl3JUk0wmPXAWKzbs5ItF69kfTvJu4U9Q+OcJhyag9997l4HnX8jg\nK65k9qyZ3HHbMN4e8y5Z9bMY/+EkamVmMn/et/z+5ht5e8y7VKuePP84JlVNm0OfvvCXMjrvCKDI\npJ1WKzGSdGGr1m2heaPaecvNGmQecpExZ+N2Lh/+EgDVqlTi/FM6smPXngqNM2hWbdpFs/oHe9ZN\n61YjZ/MPBdoMPqUNg0Z+CMD0JRuokpFG3ZqV2bR9D3e+cnCu4/H3nsHSHF34Laxho0asWXNwptC1\nOWvJympQoM3bb73BP595DoBOnbuwd+8eNm/eRJ06dcmoVAmA9sd0oFnz5ixfvoz2xyRP6S9o5YMS\n43X3T6LzjkwHJudbnkJkOsJiRR9UWdRrDtCwpH0T0fR5K2ndrD4tGtUhIz2Ni07vwrhPC45yqJt5\nMPkMu7Ifo8aW+CMS4OulG2nVsCbN61cnIy3EoJ5H8N6MlQXarNywg74dI1/vj2pSi0rpITZt30OV\njDSqVkoD4JSOjdmXGy5wAVMiOhzbkZUrVrB69Sr27d3L+Pffpe8p/Qu0adKkCV9+/hkA3y1dyt69\ne6lTpy6bN28iHI58W8xeuZKVK1fQtFnidaoOR9IN+Yv6iMhcsDuiy1WJTEN4Ygn7NCQy6ffmQusN\n+OxHxJgQwmFnyCNvM/Z/riNkkSF/C5et467rTmfG/JW8P2U+J3VtzX03DiAcdqZ88x2/f/iteIed\n8MLuDH1xGm/f3p+QGS9PWsKi1du4/aJOfL10I+O/WcVdr8zg79f15Iaz2uPuXP/PyK9PVmYV3hre\nn1x31mzaxW+enBrnT5OY0tLSuP3Ou/ntdVfjYef8QRfRqnVr/vHE3+lwbEdO7nsKfxh6GyPuuYtX\nXnoRC4X404N/BmDG9On844m/k56eTlpaiLvvuY9atWqVcsZgSZBcHDMrqt51SCOzme7epbR1hbY/\nR+TxOlOK2Paqu/+8qP2q9hiqomQ5q9S6c7xDSHprXx4c7xBSQpX0w7+h8f+NXRhzznn03KPjnuJj\n7WnvNLOu7v41gJkdD/xQ0g7ufk0J24pM2CIiFS1oPe1Yk/bvgdfN7MDVjMZExnCLiARawAaPxPwQ\nhGlm1o7IkxkMWODu+8o1MhGRCpAoN83EqrTb2Pu5+0QzG1RoU1szw911pU1EAi1oQ/5K62mfDEwE\nzi1imwNK2iISaIkylC9WJSbtfA+mvM/dv8+/zcyOLLeoREQqSMCqIzF/M3iziHVvlGUgIiLxELLY\nX4mgtJp2O6ADkFmorl0LqFKegYmIVISkuhBJZLTIOUBtCta1twPXlVdQIiIVJWA5u9Sa9jvAO2bW\n090/r6CYREQqTKKUPWIVa037AjOrZWYZZvaRma03s1+Wa2QiIhXAfsR/iSDWpH26u28jUipZBrQB\nhpVXUCIiFSU9FPsrEcR6G3tG9P8DgNfdfWvQJg4XESlK0HJZrEl7rJktIDJJ1A1mlgXsLr+wREQq\nRlLWtN19ONAT6Badc2Qnhz7VRkQkcJLtGZH5NQVOM7P847NfKuN4REQqVLKN0wbAzO4B+gLHAO8B\nZxF55JiStogEWlKWR4CLgP5AjrtfBXQGMsstKhGRCpJmFvMrEcRaHvnB3cNmtt/MagHrgOR6uqeI\npKQEycUxizVpTzez2sCzwAwiD/jVHZIiEnhBK4/E+uSaG6JvnzKzD4Ba7j67/MISEakYSXUh0sy6\nlrTtwIN+RUSCKmA5u9Se9qMlbHOgXxnGIiJS4cqip21mZwJ/JTK44zl3/3Oh7c2BUURmTA0Bw939\nAzP7OZEpQZzI83c7AceVVMkobZa/Uw7ng4iIJLrDzdlmFgKeIDLCbjUwzczecfcF+ZrdBbzm7k+b\nWXsiQ6ePdPdXgVejxzkW+E9ppecSh/yZ2a353l9caNuDsX8sEZHEVAZD/noAi919efSO8dEcesd4\nmMjDYyDS215VxHEuB/6vtHhLG6d9Wb73txfadmZpBxcRSXT2I17FaAqszLecHV2X3whgsJmtBMYB\nNxdxnEspg6RtxbwvallEJHBCZjG/ilHUBi+0fDnwgrs3B84GXilwALMewE53n1davKVdiPRi3he1\nLCISOCX1PudN/5z5Mw7ekjL42cf7uvukQs2ygRb5lpsRqW3ndw1wBoC7f2FmVcysvrtviG6/jBh6\n2VB60u5sZtuIfK6q0fdEl/VgXxEJvJIuRHbo3pMO3XvmLb/1zGOTimg2DWhjZi2BNUQS8OWF2iwH\nTgVGRS9EVj6QsC0yoffFQJ9Y4i1t9EhaLAcREQmqw30IgrvnmtlNwAQODvmbb2YjgGnuPg4YCjxr\nZkOIXJS8It8hTgJWuvuyWM73Y6ZmFRFJOmXxFDF3/wA4utC6e/K9nw/0LmbfT4ATYz2XkraIpLSk\nuo09Lhq2incESW/vgi/jHULSe+/bU+MdQkoY1LnxYR8jWZ8RKSKSlBLkIesxU9IWkZSmnraISIAE\nK2UraYtIigtYR1tJW0RSWyhgfW0lbRFJaRryJyISIAHL2UraIpLaVB4REQkQ9bRFRAJESVtEJEBM\n5RERkeAIBStnK2mLSGrTkD8RkQBReUREJEBUHhERCRD1tEVEAiRgJW0lbRFJbQHL2UraIpLa0gLW\n1VbSFpHUFqycraQtIqlNFyJFRAIkYNURJW0RSW0By9lK2iKS4gKWtZW0RSSlqaYtIhIguo1dRCRI\nlLRFRIJD5RERkQDRkD8RkQAJWM5W0haRFBewrK2kLSIpTTXtJHZa1+Y8cl1vQmaM+u98Hn3zmwLb\nm2fV4KnfnUL9zKps2rabqx/7kDWbdsUp2mA67cT2PDL0QkKhEKP+8xmPvvhhge3NG9XhqXt/Qf06\nNdi0ZSdX3/USa9ZvjVO0wbFw5pe8++IThN3pfsoATj7/5wW2b9mwjtefHMnuXTvwcJgzfvFrju7y\nM2ZO+ZBPx4zGzHB3cpZ/x80PP0vjlq3j9EnKnob8JSkzePw3JzHgrndYvWkXUx67iLFffs+i7C15\nbUZefSIvf7SQ0ZMW0efYJtx/RU+uffyjOEYdLGbG47ddwoDf/p3V67cy5ZVbGTtpDouWrc1rM3LI\nBbw85ktGvzeNPse35f7fnce1d78cx6gTXzgcZsxzf+PaPz5GrTr1efL239C+ey8aNG2Z1+bjt16m\n04mn8LPTzmNd9nJeHHkbtz45mi69T6VL71MByFnxHS8/cldSJWwgcOWRULwDCIruRzVkyZotrFi/\ng/25Yd6YvJhzf3ZkgTbtmtfhk9nZAEyeu5pzfnZEHCINru7HtmTJynWsWLOZ/fvDvDF+Buf27Vig\nTbtWjfhk2iIAJs9YzDknd4pHqIGSvWQ+9Rs3o05WI9LS0+nUqx/zp00t0MbM2PND5FvhD7t2UKtu\n/UOOM2vqRDr36l8hMVck+xH/JYJyTdpm1s7M+ptZjULrzyzP85aHJvWqk71+R95y9oadNKlXvUCb\n2d9t5PwTI72QgT1bUaNKBrWrV67QOIOsSYPaZOcc/OaSvXYLTRrULtBm9sJszu/fBYCB/TpTo1pl\natesWqFxBs22TRvIrNcgbzmzbhZbN20o0Kb/xVfwzacTeOj6ixn10O2cd/Uthxxn9mcT6dw7CZO2\nxf5KBOWWtM3sd8A7wM3AXDMbmG/zg+V13vJS1J+Xe8HlO174jJM6NmHq4xfRq0NjVm/ayf5wuELi\nSwZF/aXwQj/kO/76H07q1pap/3srvY5rzep1W9ifq59xSRw/ZJ0V+mHPmjKR4/uexfB/vs6Vw0fy\n2v88UGD7yiXzqVSlKg2bHVGeocaF/YhXsccwO9PMFpjZIjO7rYjtzc1sopl9bWYzzeysfNs6mdln\nZjbXzGaZWaWS4i3PmvZ1wPHuvsPMjgDeMLMj3P1vlPD5czcsIrxxcd5yqF5b0uofVY5hxmbVxp00\nz6qZt9ysfnXWbNpZoE3O5l1cPnI8ANUqp3P+ia3Y8cO+Co0zyFat3ULzRnXylps1rH3IRcacDdu4\nfOi/AKhWpRLn9+/Cjl17KjTOoMmsm8WWDQevC2zdtJ5adeoVaDP943e56s5HAGhxVAf279vLzm1b\nqF4r8k0nUhrpV3FBV6TD7EGbWQh4AugPrAammdk77r4gX7O7gNfc/Wkzaw+8BxxpZmnAy8Av3H2u\nmdUBSkwa5VkeSXP3HQDuvgzoC5xlZo9Rwo8prf5RZBx9dt4rERI2wPTF62jdOJMWWTXISA9xUZ+2\njPtyWYE2dWseLIUMu7gro/67AInd9G+X07p5Fi0a1yEjPY2LzjiecZ/MKdCmbubBktSwq09n1Dtf\nVHSYgdOsTTs25qxi8/oc9u/fx+ypE2nfrVeBNrXrN2LJ7BkArMtezv59e/MStrsz94tJdD4xOZN2\nGdS0ewDaGGa0AAAHgUlEQVSL3X25u+8DRgMDC7UJA7Wi72sDq6LvTwdmuftcAHff7IW/XhZSnj3t\nHDPr4u4zo8HsMLNzgOeBjiXvmnjCYWfI058y9r5zCYUiQ/4WZm/mrp93Z8bidbw/bTkndWzKfb86\ngbA7U+au5vdPTY532IESDjtD/vxvxv7jpsjP+D+fs/D7tdz12wHM+HYF70+ey0nd2nLfzecRDoeZ\n8vVSfv/Qa/EOO+GFQmmcd80tPP+nYbiH6dZvAA2ateS//36BZq2Ppv3xJzJg8PW89fQjTH33dSwU\n4uIbb8/b//v5s8is14A6DRrH8VOUnzIY8tcUWJlvOZtIIs9vBDAhWjauBpwaXX8UgJl9ANQn0ht/\npKSTWSlJ/Sczs2bAfnfPKWJbL3efWsRuVD33H+UTkByUPS/eESS9/33xzniHkBIGdW582Cl30dpd\nxeacL6d+ylefHex8PfHog6e4+6T8bczsIuB0d/91dPmXQHd3vyVfmyEA7v64mZ0APOfuHczs/wE3\nAN2A3cBHwJ3u/nFxMZVbT9vds0vYVmTCFhGpaCUN5Tuh18mc0OvkvOX/+csDk4polg20yLfcjEht\nO79rgDMA3P0LM6tiZvWj+37i7psBzOw9oCtQbNLWOG0RSWllMORvGtDGzFpGR35cBowp1GY50ZJI\n9EJkZXffAIwHOkWTeDpwMlDiV2HdESkiKe1w6yvunmtmNwETiHSEn3P3+WY2Apjm7uOAocCz0TJJ\nGLgiuu+W6OCM6dH177r7+yWdT0lbRFJbGdw04+4fAEcXWndPvvfzgd7F7Psq8Gqs51LSFpGUlii3\np8dKSVtEUppm+RMRCRIlbRGR4FB5REQkQBJl9r5YKWmLSEoLWM5W0haR1KaetohIoAQraytpi0hK\n05A/EZEAUXlERCRANORPRCRIgpWzlbRFJLUFLGcraYtIalNNW0QkQCxgWVtJW0RSWrBStpK2iKS4\ngHW0lbRFJLVpyJ+ISIAEraetp7GLiASIetoiktKC1tNW0haRlBYKWNZW0haRlBaslK2kLSKpLmBZ\nW0lbRFJa0Ib8afRIGcjdsCjeISS93O2r4h1C0ps77bN4hxAXZrG/EoGSdhkIb1wc7xCSXniHknZ5\n+3Z6iibtH/FKBCqPiEhqS5RsHCMlbRFJaUEb8mfuHu8YAs/M+rr7pHjHkcz0My5/+hkHg5K2iEiA\n6EKkiEiAKGmLiASIkvZhMLMzzWyBmS0ys9viHU8yMrPnzGytmc2OdyzJysyamdlEM5tnZnPM7Hfx\njkmKp5r2T2RmIWAR0B9YDUwDLnP3BXENLMmYWW9gB/CSu3eKdzzJyMwaAY3cfaaZ1QBmAAP1u5yY\n1NP+6XoAi919ubvvA0YDA+McU9Jx9ynA5njHkczcPcfdZ0bf7wDmA03jG5UUR0n7p2sKrMy3nI1+\n0SXgzOwIoAvwZXwjkeIoaf90RY3IV61JAitaGnkDuCXa45YEpKT902UDLfItNyNS2xYJHDNLJ5Kw\nX3b3d+IdjxRPSfunmwa0MbOWZlYJuAwYE+eYklUizdeTrJ4H5rn73+IdiJRMSfsncvdc4CZgAvAt\nMNrd58c3quRjZq8CnwFHmdkKM7sq3jElGzPrBfwC6Gdm35jZ12Z2ZrzjkqJpyJ+ISICopy0iEiBK\n2iIiAaKkLSISIEraIiIBoqQtIhIgStoiIgGipC2Hzcxyo2N755jZO2ZW6zCO9YKZfRc93nQz+1lZ\nxioSdEraUhZ2untXd+9IZEa+Gw/zeEPdvStwO/BMrDuZWdphnlck4elp7FLWPgc6Hlgws6HAJUAl\n4G13HxFdfzeRu/DWEZnHZbq7P1boWJ8CraPtrwV+DWQAS4DB7r7bzF4AdgPHAVPM7DXgr0AV4Afg\nKndfbGZXAOcD1YE2wKPRmAZH9x/g7lvK+GchUubU05ayYJDX0+1PdA4WMzsNaOvuPYgk1W5m1tvM\njgcuADoBA4BuxRz3PGBO9P2b7t7D3Y8DFgDX5GvX1N1PcPehROaC7uPuxwP3ACPztetAJHH3AB4A\ndkR79F8AvzqcH4BIRVFPW8pCVTP7mshMh/OA/0bXnw6cFt1mRHq5bYFawDvuvhfYa2ZjCx3vL2Z2\nF7Ceg8m5k5ndD9SOHmd8vvav53tfG3jJzNoSmSo3/+/4x+6+C9hlZluAcdH1c8j37UAkkamnLWVh\nV7TH2oJIcj5Q0zZgZLTefZy7H+XuL1D6jH1Do/uc4e7zouteAG6IPnLsPiLljwN25nt/PzAxWl8/\nt1C7Pfnee77lMOrASEAoaUtZMAB33w3cAgyLlkrGA1ebWXUAM2tiZlnAFOBcM6scnXj/nBjOUQPI\nMbMMIrXw4tQCVkXfa0ZASTpK2lIW8qaKjD5rcCaRhxz/F/g/4PPo09RfB2q4+3Qide9ZwLvAbGBr\n4WMVcjfwFTCZSN36kHNHPQI8ZGYzKPn3W9NbSiBpalaJCzOr7u47zawqkVEi1x14uKyIFE91PImX\nZ8zsGKAy8KIStkhs1NMWEQkQ1bRFRAJESVtEJECUtEVEAkRJW0QkQJS0RUQCRElbRCRA/j+xNceU\nqQFfiQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f55db93cc88>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.heatmap(grid_reshape, cmap='Blues', annot=True)\n",
    "plt.xlabel('RegParam')\n",
    "plt.ylabel('ElasticParam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Area under precision-recall curve - training: 0.908030706798117\n",
      "Area under precision-recall curve - testing: 0.8957374196681749\n"
     ]
    }
   ],
   "source": [
    "# evaluate the optimal model quantitatively.\n",
    "\n",
    "evaluator = BinaryClassificationEvaluator().setMetricName(\"areaUnderPR\")\n",
    "print (\"Area under precision-recall curve - training:\",evaluator.evaluate(train_predictions))\n",
    "print (\"Area under precision-recall curve - testing:\",evaluator.evaluate(test_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set accuracy: 0.7254445737078278\n"
     ]
    }
   ],
   "source": [
    "print (\"Test set accuracy:\",test_predictions.filter((test_predictions['prediction'] == test_predictions['label'])).count() / test_predictions.select(\"label\").count())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Running Experiments for Other Feature types\n",
    "Below sections fit pipelines and generate grid search results for other feature types: Ngram, Hashing Vector, Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# packages\n",
    "from pyspark.ml.linalg import Vector\n",
    "from pyspark.ml import Pipeline, PipelineModel\n",
    "\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml.classification import GBTClassifier\n",
    "\n",
    "from pyspark.ml.feature import HashingTF,StopWordsRemover,IDF,Tokenizer\n",
    "from pyspark.ml.feature import IndexToString, StringIndexer, VectorIndexer, OneHotEncoder, VectorAssembler\n",
    "\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from pyspark.ml.tuning import ParamGridBuilder, CrossValidator\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For Hashing Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_hashvector_pipeline(hv_param=500,ml_param=0):\n",
    "    #Constructing a pipeline for hash vetctors\n",
    "\n",
    "    # String to index transformer e.g. Score is a string of 1-5, but need it as an index which can be one-hot encoded\n",
    "    scoreIndexer = StringIndexer(inputCol = \"Score\", outputCol = \"indexedScore\")\n",
    "\n",
    "    # One-hot encoder on the indexed feature(s)\n",
    "    scoreEncoder = OneHotEncoder(inputCol = \"indexedScore\", outputCol = \"scoreVec\")\n",
    "\n",
    "    #We split each sentence into words using Tokenizer. \n",
    "    #Tokenizer only splits by white spaces\n",
    "    tokenizer = Tokenizer().setInputCol(\"reviewTextNoPunc\").setOutputCol(\"words\")\n",
    "\n",
    "    #Remove stopwords\n",
    "    remover= StopWordsRemover().setInputCol(\"words\").setOutputCol(\"filtered\").setCaseSensitive(False)\n",
    "\n",
    "    #For each sentence (bag of words),use HashingTF to hash the sentence into a feature vector. \n",
    "    hashingTF = HashingTF().setNumFeatures(hv_param).setInputCol(\"filtered\").setOutputCol(\"rawFeatures\")\n",
    "\n",
    "    #We use IDF to rescale the feature vectors; this generally improves performance when using text as features.\n",
    "    idf = IDF().setInputCol(\"rawFeatures\").setOutputCol(\"idfFeature\").setMinDocFreq(0)\n",
    "\n",
    "    # Assemble the vector of features\n",
    "    assembler = VectorAssembler(inputCols = [\"scoreVec\", \"idfFeature\", \"reviewCount\"], outputCol = \"features\")\n",
    "\n",
    "    #Our feature vectors could then be passed to a learning algorithm.\n",
    "    # Create a Logistic regression model\n",
    "    lr = LogisticRegression(elasticNetParam=ml_param, labelCol = \"label\")\n",
    "    \n",
    "    #Then basically we connect all the steps above to create one pipeline\n",
    "    pipeline=Pipeline(stages=[scoreIndexer, scoreEncoder, tokenizer, remover, hashingTF, idf, assembler, lr])\n",
    "    return pipeline\n",
    "\n",
    "def get_hashvector_paramgrid(value_list):\n",
    "    hashingTF = HashingTF().setNumFeatures(500).setInputCol(\"filtered\").setOutputCol(\"rawFeatures\")\n",
    "    lr = LogisticRegression(labelCol = \"label\")\n",
    "    #and associated params\n",
    "    paramGrid = ParamGridBuilder()\\\n",
    "        .addGrid(hashingTF.numFeatures,value_list)\\\n",
    "        .addGrid(lr.elasticNetParam, [0]) \\\n",
    "        .build() \\\n",
    "        #.addGrid(lr.setRegParam, lr_values)\\\n",
    "    return paramGrid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For nGrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_ngram_pipeline(ng_param=2,ml_param=0):\n",
    "    #Constructing a pipeline for hash vetctors\n",
    "\n",
    "    # String to index transformer e.g. Score is a string of 1-5, but need it as an index which can be one-hot encoded\n",
    "    scoreIndexer = StringIndexer(inputCol = \"Score\", outputCol = \"indexedScore\")\n",
    "\n",
    "    # One-hot encoder on the indexed feature(s)\n",
    "    scoreEncoder = OneHotEncoder(inputCol = \"indexedScore\", outputCol = \"scoreVec\")\n",
    "\n",
    "    #We split each sentence into words using Tokenizer. \n",
    "    #Tokenizer only splits by white spaces\n",
    "    tokenizer = Tokenizer().setInputCol(\"reviewTextNoPunc\").setOutputCol(\"words\")\n",
    "\n",
    "    #Remove stopwords\n",
    "    #remover= StopWordsRemover().setInputCol(\"words\").setOutputCol(\"filtered\").setCaseSensitive(False)\n",
    "\n",
    "    #For each sentence (bag of words), generate n-grams\n",
    "    nGram = NGram(n=ng_param, inputCol=\"words\", outputCol=\"rawFeatures\") \n",
    "    vectorizer = CountVectorizer(inputCol=\"rawFeatures\", outputCol=\"rawFeaturesvec\")\n",
    "\n",
    "    #We use IDF to rescale the feature vectors; this generally improves performance when using text as features.\n",
    "    idf = IDF().setInputCol(\"rawFeaturesvec\").setOutputCol(\"idfFeature\").setMinDocFreq(0)\n",
    "\n",
    "    # Assemble the vector of features\n",
    "    assembler = VectorAssembler(inputCols = [\"scoreVec\", \"idfFeature\", \"reviewCount\"], outputCol = \"features\")\n",
    "\n",
    "    #Our feature vectors could then be passed to a learning algorithm.\n",
    "    # Create a Logistic regression model\n",
    "    lr = LogisticRegression(elasticNetParam=ml_param, labelCol = \"label\")\n",
    "\n",
    "    #Then basically we connect all the steps above to create one pipeline\n",
    "    pipeline=Pipeline(stages=[scoreIndexer, scoreEncoder, tokenizer, nGram, vectorizer, idf,assembler, lr])\n",
    "    return pipeline\n",
    "\n",
    "def get_ngram_paramgrid(value_list):\n",
    "    nGram = NGram(n=2, inputCol=\"filtered\", outputCol=\"rawFeatures\") \n",
    "    lr = LogisticRegression(labelCol = \"label\")\n",
    "    #and associated params\n",
    "    paramGrid = ParamGridBuilder()\\\n",
    "        .addGrid(nGram.n,value_list)\\\n",
    "        .addGrid(lr.elasticNetParam, [0]) \\\n",
    "        .build() \\\n",
    "        #.addGrid(lr.setRegParam, lr_values)\\\n",
    "    return paramGrid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_w2v_pipeline(w2_param=3,ml_param=0):\n",
    "    #Constructing a pipeline for Word2Vector\n",
    "\n",
    "    # String to index transformer e.g. Score is a string of 1-5, but need it as an index which can be one-hot encoded\n",
    "    scoreIndexer = StringIndexer(inputCol = \"Score\", outputCol = \"indexedScore\")\n",
    "\n",
    "    # One-hot encoder on the indexed feature(s)\n",
    "    scoreEncoder = OneHotEncoder(inputCol = \"indexedScore\", outputCol = \"scoreVec\")\n",
    "\n",
    "    #We split each sentence into words using Tokenizer. \n",
    "    #Tokenizer only splits by white spaces\n",
    "    tokenizer = Tokenizer().setInputCol(\"reviewTextNoPunc\").setOutputCol(\"words\")\n",
    "\n",
    "    #Remove stopwords\n",
    "    #remover= StopWordsRemover().setInputCol(\"words\").setOutputCol(\"filtered\").setCaseSensitive(False)\n",
    "\n",
    "    #For each sentence (bag of words), generate word2vec representation\n",
    "    w2v = Word2Vec(vectorSize=w2_param, minCount=0, inputCol=\"words\", outputCol=\"rawfeatures\")\n",
    "\n",
    "    #We use IDF to rescale the feature vectors; this generally improves performance when using text as features.\n",
    "    idf = IDF().setInputCol(\"rawfeatures\").setOutputCol(\"idfFeature\").setMinDocFreq(0)\n",
    "\n",
    "    # Assemble the vector of features\n",
    "    assembler = VectorAssembler(inputCols = [\"scoreVec\", \"idfFeature\", \"reviewCount\"], outputCol = \"features\")\n",
    "\n",
    "    #Our feature vectors could then be passed to a learning algorithm.\n",
    "    # Create a Logistic regression model\n",
    "    lr = LogisticRegression(elasticNetParam=ml_param, labelCol = \"label\")\n",
    "\n",
    "    #Then basically we connect all the steps above to create one pipeline\n",
    "    pipeline=Pipeline(stages=[scoreIndexer, scoreEncoder, tokenizer, w2v, idf, assembler, lr])\n",
    "    return pipeline\n",
    "\n",
    "\n",
    "def get_w2v_paramgrid(value_list):\n",
    "    w2v = Word2Vec(vectorSize=3, minCount=0, inputCol=\"words\", outputCol=\"features\")\n",
    "    lr = LogisticRegression(labelCol = \"label\")\n",
    "    #and associated params\n",
    "    paramGrid = ParamGridBuilder()\\\n",
    "        .addGrid(w2v.vectorSize, value_list)\\\n",
    "        .addGrid(lr.elasticNetParam, [0]) \\\n",
    "        .build()\n",
    "        #.addGrid(lr.setRegParam, lr_values)\\\n",
    "    return paramGrid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now actually run grid search over different feature types\n",
    "Print results to dataframe csv to allow graphics generation in a different notebook: goal is to be able to generate various graphics from grid search results without having to recalculate the whole thing\n",
    "\n",
    "Note: code  changed to run over dimensions of the grid search sequentially rather than in parallel: this was done because the whole grid search kept crashing out partway due to VPN connection issues and competition for server memory with other users: safer to print it on each loop!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First define some functions \n",
    "For interpreting parameter grids and recording CPU time usage etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "from io import TextIOWrapper, BytesIO\n",
    "\n",
    "def get_all_times(command, args):\n",
    "    # setup the environment\n",
    "    old_stdout = sys.stdout\n",
    "    sys.stdout = TextIOWrapper(BytesIO(), sys.stdout.encoding)\n",
    "\n",
    "    %time model=command(*args)\n",
    "    sys.stdout.seek(0)\n",
    "    out = sys.stdout.read()\n",
    "\n",
    "    sys.stdout.close()\n",
    "    sys.stdout = old_stdout\n",
    "\n",
    "    times = out\n",
    "    \n",
    "    try:\n",
    "        model=command(*args)\n",
    "        return (model,times)\n",
    "    except Exception as ex:\n",
    "        print(ex)\n",
    "        return (None,times)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_param_df(paramGrid, metrics=[]):\n",
    "    params_df = pd.DataFrame(index=[])\n",
    "\n",
    "    for i, param in enumerate(paramGrid):\n",
    "        col_names=[]\n",
    "        param_values=[]\n",
    "        for param_name in param.keys():\n",
    "            col_name = param_name.parent.split('_')[0] + '_' + param_name.name \n",
    "            col_names.append(col_name)\n",
    "            param_value = param[param_name]\n",
    "            param_values.append(param_value)\n",
    "            \n",
    "        this_result=pd.DataFrame([param_values], columns=col_names)\n",
    "        if metrics != []:\n",
    "            this_result['metric'] = metrics[i]\n",
    "        params_df = pd.concat([params_df,this_result],axis=0)\n",
    "    return params_df    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finally, run the full grid search over training set sizes, param type, param features\n",
    "\n",
    "Note: This gridsearch outputs to a results csv which is designed to be read by the Graphics notebook, not read here!!!\n",
    "\n",
    "The reason for this is to be able to pivot the results in a more agile way without rerunning any grid searches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running training set size: 0.01\n",
      "Processing feature type word2vec\n",
      "Pipeline: Pipeline_4d1f914eacf2dcf3d672\n",
      "Paramgrid values: [10]\n",
      "Paramgrid: [{Param(parent='Word2Vec_470fbc2c943d243e62b4', name='vectorSize', doc='the dimension of codes after transforming from words'): 10, Param(parent='LogisticRegression_48ad9c9fc73a1e2168d3', name='elasticNetParam', doc='the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.'): 0}]\n"
     ]
    }
   ],
   "source": [
    "#set config here: currently set to very small grid search dimensions for testing purposes\n",
    "\n",
    "feat_types = ['word2vec','hashing_vector','ngram']\n",
    "pipeline_dict = {'hashing_vector':get_hashvector_pipeline, 'ngram' : get_ngram_pipeline, 'word2vec' : get_w2v_pipeline }\n",
    "param_dict = {'hashing_vector':get_hashvector_paramgrid, 'ngram' : get_ngram_paramgrid, 'word2vec' : get_w2v_paramgrid }\n",
    "values_dict = {'hashing_vector':[[10],[100],[1000],[10000]], 'word2vec':[[10], [100],[1000]], 'ngram':[[2]]}\n",
    "\n",
    "el_param=0\n",
    "\n",
    "#train_size = [0.10, .20, .30, .40, .50, .60, .70, .80, .90, 0.99]\n",
    "train_sizes = [0.01]\n",
    "#train_sizes = [0.5,0.99]\n",
    "\n",
    "models=[]\n",
    "times_list=[]\n",
    "results_df=pd.DataFrame(index=[])\n",
    "\n",
    "for train_size in train_sizes:\n",
    "    print('Running training set size: {}'.format(train_size))\n",
    "    train_set_sampled = train_set.sample(False, train_size, seed=123)\n",
    "    train_set_sampled.persist(StorageLevel.MEMORY_AND_DISK)\n",
    "    for feat in feat_types:\n",
    "        print('Processing feature type {}'.format(feat))\n",
    "        \n",
    "        for paramgrid_values in values_dict[feat]:\n",
    "            pipeline = pipeline_dict[feat](paramgrid_values[0], el_param)\n",
    "            print('Pipeline: {}'.format(pipeline))\n",
    "            #paramgrid_values = values_dict[feat]\n",
    "            print('Paramgrid values: {}'.format(paramgrid_values))\n",
    "            paramGrid = param_dict[feat](paramgrid_values)\n",
    "            print('Paramgrid: {}'.format(paramGrid))\n",
    "\n",
    "            tvs = TrainValidationSplit(estimator=pipeline,\n",
    "                                   estimatorParamMaps=paramGrid,\n",
    "                                   evaluator=BinaryClassificationEvaluator().setMetricName(\"areaUnderPR\"),\n",
    "                                   # 80% of the data will be used for training, 20% for validation.\n",
    "                                   trainRatio=0.8)\n",
    "\n",
    "            tuned_model,time = get_all_times(tvs.fit,(train_set_sampled,)) \n",
    "            models.append(tuned_model)\n",
    "            times_list.append(time)\n",
    "            print('AUC returned: {}'.format(tuned_model.validationMetrics))\n",
    "            this_result_df = get_param_df(paramGrid,tuned_model.validationMetrics)\n",
    "            this_result_df['calc_time'] = time\n",
    "            this_result_df['feature_type'] = feat\n",
    "            this_result_df['train_set_size'] = train_size\n",
    "            results_df = pd.concat([results_df,this_result_df],axis=0)\n",
    "            results_df.to_csv('gridsearch_results.csv',sep='#')\n",
    "\n",
    "            #tuned_model,times = tvs2.fit(train_set)\n",
    "            print('Feature type {} got tuned model: {} and time results: {}'.format(feat, tuned_model, time))\n",
    "\n",
    "print(results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark",
   "language": "python",
   "name": "pyspark"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}