Fixing python3 (#192)

* Various python3 updates.

* XMLParser can't be modified in Python3.

* Updating for python3.

* xmlUtils does not always have a python path for utils.

* Making the necessary edits to allow xmlUtils to import utils from the writeTex.py module.

* Do a decode of the output to ascii, instead of comparing to binary.

* Switching the element tree parser in the TreeStructure.py module to use a method that handles comments equally well for Python 2 and 3.

At this point, we are now passing 103 test cases with python 3, but more work needs to be done in order to have full compliance with python 2 and 3.

crow/setup3.py

@@ -14,7 +14,7 @@
 from distutils.core import setup, Extension
 import subprocess
 try:
-  eigen_flags = subprocess.check_output(["./scripts/find_eigen.py"])
+  eigen_flags = subprocess.check_output(["./scripts/find_eigen.py"]).decode("ascii")
 except:
   eigen_flags = ""
 include_dirs=['include/distributions','include/utilities','contrib/include']
@@ -26,7 +26,7 @@
 setup(name='crow',
       version='0.8',
       ext_package='crow_modules',
-      ext_modules=[Extension('_distribution1D'+ext,['crow_modules/distribution1D'+ext+'.i','src/distributions/distribution.C','src/utilities/MDreader.C','src/utilities/inverseDistanceWeigthing.C','src/utilities/microSphere.C','src/utilities/NDspline.C','src/utilities/ND_Interpolation_Functions.C','src/distributions/distributionNDBase.C','src/distributions/distributionNDNormal.C','src/distributions/distributionFunctions.C','src/distributions/DistributionContainer.C','src/distributions/distribution_1D.C','src/distributions/randomClass.C','src/distributions/distributionNDCartesianSpline.C'],include_dirs=include_dirs,swig_opts=swig_opts,extra_compile_args=extra_compile_args),
-                   Extension('_interpolationND'+ext,['crow_modules/interpolationND'+ext+'.i','src/utilities/ND_Interpolation_Functions.C','src/utilities/NDspline.C','src/utilities/microSphere.C','src/utilities/inverseDistanceWeigthing.C','src/utilities/MDreader.C','src/distributions/randomClass.C'],include_dirs=include_dirs,swig_opts=swig_opts,extra_compile_args=extra_compile_args)],
+      ext_modules=[Extension('_distribution1D'+ext,['crow_modules/distribution1D'+ext+'.i','src/distributions/distribution.cxx','src/utilities/MDreader.cxx','src/utilities/inverseDistanceWeigthing.cxx','src/utilities/microSphere.cxx','src/utilities/NDspline.cxx','src/utilities/ND_Interpolation_Functions.cxx','src/distributions/distributionNDBase.cxx','src/distributions/distributionNDNormal.cxx','src/distributions/distributionFunctions.cxx','src/distributions/DistributionContainer.cxx','src/distributions/distribution_1D.cxx','src/distributions/randomClass.cxx','src/distributions/distributionNDCartesianSpline.cxx'],include_dirs=include_dirs,swig_opts=swig_opts,extra_compile_args=extra_compile_args),
+                   Extension('_interpolationND'+ext,['crow_modules/interpolationND'+ext+'.i','src/utilities/ND_Interpolation_Functions.cxx','src/utilities/NDspline.cxx','src/utilities/microSphere.cxx','src/utilities/inverseDistanceWeigthing.cxx','src/utilities/MDreader.cxx','src/distributions/randomClass.cxx'],include_dirs=include_dirs,swig_opts=swig_opts,extra_compile_args=extra_compile_args)],
       py_modules=['crow_modules.distribution1D'+ext,'crow_modules.interpolationND'+ext],
       )

doc/user_guide/writeTex.py

@@ -15,8 +15,8 @@
 import sys, os, time
 import xml.etree.ElementTree as ET
 #load XML navigation tools
-sys.path.append(os.path.join(os.getcwd(),'..','..','framework','utils'))
-import xmlUtils
+sys.path.append(os.path.join(os.getcwd(),'..','..','framework'))
+from utils import xmlUtils
 
 def getNode(fname,nodepath):
   """

framework/CodeInterfaces/MELCOR/melgenInterface.py

@@ -20,7 +20,6 @@
 warnings.simplefilter('default',DeprecationWarning)
 
 import os
-from  __builtin__ import any as bAny
 import GenericParser
 from utils import utils
 from CodeInterfaceBaseClass import CodeInterfaceBase
@@ -116,7 +115,7 @@ def checkForOutputFailure(self,output,workingDir):
       return failure
     readLines = outputToRead.readlines()
     for goodMsg in errorWord:
-      if bAny(goodMsg in x for x in readLines):
+      if any(goodMsg in x for x in readLines):
         failure = False
         break
     return failure

framework/CodeInterfaces/RELAP5/Relap5Interface.py

@@ -25,7 +25,6 @@
 import relapdata
 import shutil
 import re
-from  __builtin__ import any as bAny
 from CodeInterfaceBaseClass import CodeInterfaceBase
 
 class Relap5(CodeInterfaceBase):
@@ -107,7 +106,6 @@ def checkForOutputFailure(self,output,workingDir):
       @ In, workingDir, string, current working dir
       @ Out, failure, bool, True if the job is failed, False otherwise
     """
-    from  __builtin__ import any as bAny
     failure = True
     errorWord = ["Transient terminated by end of time step cards","Transient terminated by trip"]
     try:
@@ -116,7 +114,7 @@ def checkForOutputFailure(self,output,workingDir):
       return failure
     readLines = outputToRead.readlines()
     for goodMsg in errorWord:
-      if bAny(goodMsg in x for x in readLines):
+      if any(goodMsg in x for x in readLines):
         failure = False
         break
     return failure

framework/CustomCommandExecuter.py

@@ -43,4 +43,4 @@ def execCommand(commandString,self=None,object=None):
     @ In, object, instance, optional, object instance
     @ Out, None
   """
-  execCommandReturn(commandString,self,object)
+  exec(commandString)

framework/DataObjects/Data.py

@@ -33,6 +33,7 @@
 import ast
 import copy
 import numpy as np
+import itertools
 from scipy import spatial
 import xml.etree.ElementTree as ET
 #External Modules End--------------------------------------------------------------------------------
@@ -639,7 +640,7 @@ def getParam(self,typeVar,keyword,nodeId=None,serialize=False):
     else:
       if typeVar.lower() in ['input','inputs','unstructuredInput']:
         returnDict = {}
-        if keyword in self._dataContainer['inputs'].keys() + self._dataContainer['unstructuredInputs'].keys():
+        if keyword in itertools.chain(self._dataContainer['inputs'].keys(),self._dataContainer['unstructuredInputs'].keys()):
           returnDict[keyword] = {}
           if self.type == 'HistorySet':
             for key in self._dataContainer['inputs'][keyword].keys():

framework/DataObjects/PointSet.py

@@ -157,7 +157,7 @@ def _updateSpecializedInputValue(self,name,value,options=None):
       #self._dataContainer['inputs'][name] = c1darray(values=np.atleast_1d(np.atleast_1d(value)[-1])) if not acceptArrayRealizations else c1darray(values=np.atleast_1d(np.atleast_1d(value)))
       self.addNodeInTreeMode(tsnode,options)
     else:
-      if name in self._dataContainer['inputs'].keys()+self._dataContainer['unstructuredInputs'].keys():
+      if name in itertools.chain(self._dataContainer['inputs'].keys(),self._dataContainer['unstructuredInputs'].keys()):
         #popped = self._dataContainer['inputs'].pop(name)
         if not unstructuredInput:
           self._dataContainer['inputs'][name].append(np.atleast_1d(np.ravel(value)[-1]))

framework/PostProcessors/BasicStatistics.py

@@ -748,7 +748,7 @@ def startMetric(metric):
     metric = 'samples'
     startMetric(metric)
     for targetP in needed[metric]:
-      calculations[metric][targetP] = len(input['targets'].values()[0])
+      calculations[metric][targetP] = len(utils.first(input['targets'].values()))
     #
     # expected value
     #

framework/Samplers/Sampler.py

@@ -295,7 +295,7 @@ def _readMoreXMLbase(self,xmlNode):
           self.raiseAnError(IOError,'The following are duplicated variables indices listed in the manifestVariablesIndex: ' + str(dups))
         listElement = self.distributions2variablesMapping[dist]
         for var in listElement:
-          self.variables2distributionsMapping[var.keys()[0]]['totDim'] = maxDim #reset the totDim to reflect the totDim of original input space
+          self.variables2distributionsMapping[utils.first(var.keys())]['totDim'] = maxDim #reset the totDim to reflect the totDim of original input space
         tempListElement = {k.strip():v for x in listElement for ks,v in x.items() for k in list(ks.strip().split(','))}
         listIndex = []
         for var in listLatentElement:

framework/SupervisedLearning.py

@@ -22,11 +22,16 @@
 #for future compatibility with Python 3--------------------------------------------------------------
 from __future__ import division, print_function, unicode_literals, absolute_import
 import warnings
-from numpy import average
-from crow_modules.distribution1Dpy2 import CDF
 warnings.simplefilter('default',DeprecationWarning)
 #End compatibility block for Python 3----------------------------------------------------------------
 
+from numpy import average
+import sys
+if sys.version_info.major > 2:
+  from crow_modules.distribution1Dpy3 import CDF
+else:
+  from crow_modules.distribution1Dpy2 import CDF
+
 #External Modules------------------------------------------------------------------------------------
 from sklearn import linear_model
 from sklearn import svm

framework/utils/TreeStructure.py

@@ -66,7 +66,8 @@ def parse(inFile,dType=None):
       raise InputParsingError('Unrecognized file type for:',inFile,' | Expected .xml, .i, .in, or .inp')
   if dType.lower()=='xml':
     #try:
-    xmltree = ET.parse(inFile,parser=XMLCommentParser()) #parser is defined below, under XMLCommentParser
+    parser = ET.XMLParser(target=CommentedTreeBuilder())
+    xmltree = ET.parse(inFile,parser=parser) #parser is defined below, under XMLCommentParser
     tree = xmlToInputTree(xmltree)
     #except Exception as e:
     #  print('ERROR: Input parsing error!')
@@ -321,32 +322,32 @@ def addChild(gNode,tNode,depth=0):
 ###########
 # PARSERS #
 ###########
-class XMLCommentParser(ET.XMLTreeBuilder):
+## Extracted from: https://stackoverflow.com/questions/33573807/faithfully-preserve-comments-in-parsed-xml-python-2-7
+## As mentioned on this post, we could also potentially use lxml to handle this
+## automatically without the need for a special class
+class CommentedTreeBuilder(ET.TreeBuilder):
   """
-    An XML parser that expands on the default to preserves comments
+      A class for preserving comments faithfully when parsing XML
   """
-  def __init__(self):
+  def __init__(self, *args, **kwargs):
     """
-      Constructor.
-      @ In, None
-      @ Out, None
+        The constructor that passes arguments to the
+        xml.etree.ElementTree.TreeBuilder class.
+        See the relevant documentation for the arguments accepted by the
+        base class's __init__ function.
     """
-    ET.XMLTreeBuilder.__init__(self)
-    self._parser.CommentHandler = self.handleComment
+    super(CommentedTreeBuilder, self).__init__(*args, **kwargs)
 
-  def handleComment(self,data):
+  def comment(self, data):
     """
-      Constructor.
-      @ In, data, string object to parse into comment
-      @ Out, None
+        A function for appropriately surrounding data with the necessary
+        markers to establish it as a comment.
+        @ In, data, string, the text that needs to be wrapped in a comment.
+        @ Out, None
     """
-    self._target.start(ET.Comment,{})
-    self._target.data(data)
-    self._target.end(ET.Comment)
-
-#set up a parser for this module
-parser = XMLCommentParser()
-
+    self.start(ET.Comment, {})
+    self.data(data)
+    self.end(ET.Comment)
 
 #########
 # NODES #

framework/utils/__init__.py

@@ -15,6 +15,7 @@
 Created on September 16, 2015
 @author: maljdp
 """
+from __future__ import absolute_import
 # This file is necessary so that the sub-modules understand the correct hierarchy
 # of things. Once everything is in sub-modules we can possibly do some things
 # with RAVEN in its entirety as a module, but for now this file can remain

framework/utils/utils.py

@@ -385,6 +385,14 @@ def toBytes(s):
   else:
     return s
 
+def isString(s):
+  """
+    Method to figure out if a variable is a string.
+    @ In, s, object, variable to be tested.
+    @ Out, isString, bool, true if variable is a str or unicode.
+  """
+  return type(s).__name__ in ['unicode','str']
+
 def toBytesIterative(s):
   """
     Method aimed to convert all the string-compatible content of

framework/utils/xmlUtils.py

@@ -25,6 +25,7 @@
 import xml.dom.minidom as pxml
 import re
 import os
+from .utils import isString
 
 #define type checking
 def isComment(node):
@@ -206,16 +207,16 @@ def fixXmlText(msg):
     @ Out, msg, string, fixed string
   """
   #if not a string, pass it back through
-  if not isinstance(msg,basestring):
+  if not isString(msg):
     return msg
   #otherwise, replace illegal characters with "?"
   # from http://boodebr.org/main/python/all-about-python-and-unicode#UNI_XML
   RE_XML_ILLEGAL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \
                  u'|' + \
                  u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \
-                  (unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
-                   unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
-                   unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff))
+                  ('\ud800','\udbff','\udc00','\udfff',
+                   '\ud800','\udbff','\udc00','\udfff',
+                   '\ud800','\udbff','\udc00','\udfff')
   msg = re.sub(RE_XML_ILLEGAL, "?", msg)
   return msg
 
@@ -226,7 +227,7 @@ def fixXmlTag(msg):
     @ Out, msg, string, fixed string
   """
   #if not a string, pass it back through
-  if not isinstance(msg,basestring):
+  if not isString(msg):
     return msg
   #define some presets
   letters = u'([a-zA-Z])'

setup3.py

@@ -13,6 +13,22 @@
 # limitations under the License.
 from distutils.core import setup, Extension
 from distutils.command.build import build
+import os
+
+# Replicating the methods used in the RAVEN Makefile to find CROW_DIR,
+# If the Makefile changes to be more robust, so should this
+# We should be doing a search for CROW, I would think, we should not force a
+# directory structure
+CURR_DIR = os.path.dirname(os.path.realpath(__file__))
+HERD_TRUNK_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),'..')
+CROW_SUBMODULE = os.path.join(CURR_DIR,'crow')
+if os.path.isfile(os.path.join(CROW_SUBMODULE,'Makefile')):
+  CROW_DIR = CROW_SUBMODULE
+else:
+  CROW_DIR = os.path.join(HERD_TRUNK_DIR,'crow')
+
+BOOST_INCLUDE_DIR = os.path.join(CROW_DIR,'contrib','include')
+RAVEN_INCLUDE_DIR = os.path.join('include','contrib')
 
 # We need a custom build order in order to ensure that amsc.py is available
 # before we try to copy it to the target location
@@ -22,15 +38,16 @@ class CustomBuild(build):
                     ('build_clib', build.has_c_libraries),
                     ('build_scripts', build.has_scripts)]
 
-include_dirs=['include/contrib']
-swig_opts=['-c++','-Iinclude/contrib','-py3']
+include_dirs=[RAVEN_INCLUDE_DIR,BOOST_INCLUDE_DIR]
+swig_opts=['-c++','-I'+RAVEN_INCLUDE_DIR, '-I'+BOOST_INCLUDE_DIR,'-py3']
+extra_compile_args=['-std=c++11']
 setup(name='amsc',
       version='0.0',
       description='A library for computing the Approximate Morse-Smale Complex (AMSC)',
       ext_modules=[Extension('_amsc',['src/contrib/amsc.i',
                                       'src/contrib/UnionFind.cpp',
                                       'src/contrib/AMSC.cpp'],
-                             include_dirs=include_dirs, swig_opts=swig_opts)],
+                             include_dirs=include_dirs, swig_opts=swig_opts,extra_compile_args=extra_compile_args)],
       package_dir={'':'src/contrib'},
       py_modules=['amsc'],
       cmdclass={'build': CustomBuild})