chapters 1 and 2 written.

pdbg · Mar 17, 2017 · 374450a · 374450a
1 parent a024716
commit 374450a
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 8 deletions.
diff --git a/courses/unstructured/.gitignore b/courses/unstructured/.gitignore
@@ -0,0 +1 @@
+tmp
diff --git a/courses/unstructured/ML-Tests-Solution.ipynb b/courses/unstructured/ML-Tests-Solution.ipynb
@@ -93,7 +93,7 @@
     "\n",
     "# Running Vision API\n",
     "import base64\n",
-    "IMAGE=\"gs://cpb103-public-files/noirbree.jpg\"\n",
+    "IMAGE=\"gs://BUCKET_NAME/unstructured/noirbree.jpg\"\n",
     "vservice = build('vision', 'v1', developerKey=APIKEY)\n",
     "request = vservice.images().annotate(body={\n",
     "        'requests': [{\n",
@@ -160,7 +160,7 @@
     }
    ],
    "source": [
-    "alice = sc.textFile(\"gs://cpb103-public-files/alice-short-transformed.txt\")\n",
+    "alice = sc.textFile(\"gs://BUCKET_NAME/unstructured/alice-short-transformed.txt\")\n",
     "alice = alice.map(lambda x: x.split(\".\"))\n",
     "                  \n",
     "for eachSentence in alice.take(10):\n",

diff --git a/courses/unstructured/PySpark-Test-Solution.ipynb b/courses/unstructured/PySpark-Test-Solution.ipynb
@@ -61,7 +61,7 @@
     "from pyspark.sql.types import *\n",
     "header = 'animal,name'\n",
     "schema = StructType([StructField(colname, StringType(), True) for colname in header.split(',')])\n",
-    "pets = spark.read.schema(schema).csv('gs://cpb103-public-files/pets.txt')\n",
+    "pets = spark.read.schema(schema).csv('gs://BUCKET_NAME/unstructured/pets.txt')\n",
     "\n",
     "pets.createOrReplaceTempView('pets')\n",
     "countsByPet = spark.sql('SELECT animal, COUNT(*) from pets GROUP BY animal')\n",
@@ -85,7 +85,7 @@
     }
    ],
    "source": [
-    "file = sc.textFile(\"gs://cpb103-public-files/pets.txt\")\n",
+    "file = sc.textFile(\"gs://BUCKET_NAME/unstructured/pets.txt\")\n",
     "\n",
     "pets = file.map(lambda s: s.split(\",\")).map(lambda x : (x[0], [x[1]]))\n",
     "petsByType = pets.reduceByKey(lambda a, b: a + b)\n",

diff --git a/courses/unstructured/lab2a-input.txt → courses/unstructured/lab2-input.txt b/courses/unstructured/lab2a-input.txt → courses/unstructured/lab2-input.txt
diff --git a/courses/unstructured/lab2a.py → courses/unstructured/lab2.py b/courses/unstructured/lab2a.py → courses/unstructured/lab2.py
@@ -3,12 +3,12 @@
 from pyspark import SparkContext
 sc = SparkContext("local")
 
-rdd = sc.parallelize(range(1000), 10)
-print rdd.mean()
-
-file = sc.textFile("gs://cpb103-public-files/lab2a-input.txt")
+file = sc.textFile("gs://BUCKET_NAME/unstructured/lab2-input.txt")
 dataLines = file.map(lambda s: s.split(",")).map(lambda x : (x[0], [x[1]]))
 print dataLines.take(100)
 
 databyKey = dataLines.reduceByKey(lambda a, b: a + b)
 print databyKey.take(100)
+
+countByKey = databyKey.map(lambda (k,v): (k, len(v)))
+print countByKey.take(100)
diff --git a/courses/unstructured/replace_and_upload.sh b/courses/unstructured/replace_and_upload.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+if [ "$#" -ne 1 ]; then
+   echo "Usage:  ./replace_and_upload.sh bucket-name"
+   exit
+fi
+
+BUCKET=$1
+echo "replacing bucket references to $BUCKET and copying to gs://$BUCKET/unstructured"
+
+# replace
+TEMP=tmp
+rm -rf $TEMP
+mkdir $TEMP
+for FILE in $(ls -1 *.py *.ipynb); do
+    echo $FILE
+    cat $FILE | sed "s/BUCKET_NAME/$BUCKET/g" > $TEMP/$FILE
+done 
+
+# first the originals, then the modified
+gsutil -m cp * gs://$BUCKET/unstructured
+gsutil -m cp $TEMP/* gs://$BUCKET/unstructured