Make yarn.py and launcher.py compatible with Python 3

tracker/dmlc_tracker/launcher.py

@@ -1,10 +1,13 @@
 #!/usr/bin/env python3
 # pylint: disable=invalid-name
 """The container launcher script that launches DMLC with the right env variable."""
+from __future__ import absolute_import
+
 import glob
 import sys
 import os
 import subprocess
+from .util import py_str
 
 def unzip_archives(ar_list, env):
     for fname in ar_list:
@@ -52,6 +55,7 @@ def main():
         (classpath, _) = subprocess.Popen('%s/bin/hadoop classpath' % hadoop_home,
                                           stdout=subprocess.PIPE, shell=True,
                                           env=os.environ).communicate()
+        classpath = py_str(class_path)
         for f in classpath.split(':'):
             class_path += glob.glob(f)
 

tracker/dmlc_tracker/util.py

@@ -0,0 +1,13 @@
+import sys
+
+# Compatibility shim for handling strings
+PY3 = (sys.version_info[0] == 3)
+
+if PY3:
+    def py_str(x):
+        """convert c string back to python string"""
+        return x.decode('utf-8')
+else:
+    def py_str(x):
+        """convert c string back to python string"""
+        return x

tracker/dmlc_tracker/yarn.py

@@ -5,13 +5,15 @@
 # pylint: disable=invalid-name, too-many-locals, too-many-branches, missing-docstring
 from __future__ import absolute_import
 import os
+import sys
 import subprocess
 import warnings
 import logging
 import platform
 from threading import Thread
 from . import opts
 from . import tracker
+from .util import py_str
 
 def yarn_submit(args, nworker, nserver, pass_env):
     """Submission function for YARN."""
@@ -45,12 +47,12 @@ def yarn_submit(args, nworker, nserver, pass_env):
     # detech hadoop version
     (out, _) = subprocess.Popen('%s version' % hadoop_binary,
                                 shell=True, stdout=subprocess.PIPE).communicate()
-    out = out.split('\n')[0].split()
+    out = py_str(out).split('\n')[0].split()
     assert out[0] == 'Hadoop', 'cannot parse hadoop version string'
     hadoop_version = int(out[1].split('.')[0])
     (classpath, _) = subprocess.Popen('%s classpath' % hadoop_binary,
                                       shell=True, stdout=subprocess.PIPE).communicate()
-    classpath = classpath.strip()
+    classpath = py_str(classpath).strip()
 
     if hadoop_version < 2:
         raise RuntimeError('Hadoop Version is %s, dmlc_yarn will need Yarn(Hadoop 2.0)' % out[1])