basic mapper

hadoop-tag.sh

@@ -0,0 +1,14 @@
+
+source ./hadoop-streaming-env.sh
+
+corpus=$1
+
+output=/sampa/home/bdmyers/nltk-apps/out
+rm -rf $output
+
+$HADOOP_HOME/bin/hadoop  jar $HADOOP_HOME/$RELATIVE_PATH_JAR \
+    -D mapred.reduce.tasks=0 \
+    -verbose \
+    -input /sampa/home/bdmyers/nltk-apps/$1 \
+    -output $output \
+    -mapper /sampa/home/bdmyers/nltk-apps/tagger_map.py 

tagger_map.py

@@ -0,0 +1,17 @@
+#!/sampa/home/bdmyers/escience/python/install/bin/python
+import sys
+import os
+import nltk
+import re
+
+# get document name
+docname = os.environ["mapreduce_map_input_file"]
+
+sent_delims = re.compile('|'.join(['\.', ';']))
+
+sentences = re.split(sent_delims, sys.stdin.read().replace('\n', ' '))[:-1]
+
+for s in sentences:
+  tokens = nltk.word_tokenize(s)
+  tagged = nltk.pos_tag(tokens)
+  print "%s\t%s" % (docname, tagged)