Skip to content

Commit

Permalink
mappers/reducers now invoked with correct env vars
Browse files Browse the repository at this point in the history
mappers and reducers are now invoked on nodes through a
script (invoke.sh) that sets the appropriate environment variables so
the correct version of python and linuxbrew packages are available
  • Loading branch information
zbsimon committed Apr 24, 2015
1 parent 2332442 commit 32a9d25
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 3 deletions.
6 changes: 6 additions & 0 deletions invoke.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/usr/bin/env bash

PATENT_SETTINGS_FILE=/shared/patents/settings.sh
source $PATENT_SETTINGS_FILE

exec $@
6 changes: 3 additions & 3 deletions map_reduce_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ def run_map_job(mapper, input_dir, output_dir,
-D stream.map.output.field.separator={2} \
-input {3} \
-output {4} \
-mapper "$NLTK_HOME/{5}" \
-mapper "./invoke.sh $NLTK_HOME/{5}" \
-inputformat {6} \
-outputformat {7}
'''.format(map_file + ",$AVRO_JAR,$HADOOP_JAR", "$AVRO_JAR,$HADOOP_JAR",
Expand Down Expand Up @@ -94,8 +94,8 @@ def run_map_reduce_job(mapper, reducer, input_dir, output_dir,
-files {0} \
-libjars {1} \
-D stream.map.output.field.separator={2} \
-mapper "$NLTK_HOME/{3}" \
-reducer "$NLTK_HOME/{4}" \
-mapper "./invoke.sh $NLTK_HOME/{3}" \
-reducer "./invoke.sh $NLTK_HOME/{4}" \
-input {5} \
-output {6} \
-inputformat {7} \
Expand Down

0 comments on commit 32a9d25

Please sign in to comment.