diff --git a/crow/setup3.py b/crow/setup3.py index 5e88151ddd..f5d2b2590a 100644 --- a/crow/setup3.py +++ b/crow/setup3.py @@ -14,7 +14,7 @@ from distutils.core import setup, Extension import subprocess try: - eigen_flags = subprocess.check_output(["./scripts/find_eigen.py"]) + eigen_flags = subprocess.check_output(["./scripts/find_eigen.py"]).decode("ascii") except: eigen_flags = "" include_dirs=['include/distributions','include/utilities','contrib/include'] @@ -26,7 +26,7 @@ setup(name='crow', version='0.8', ext_package='crow_modules', - ext_modules=[Extension('_distribution1D'+ext,['crow_modules/distribution1D'+ext+'.i','src/distributions/distribution.C','src/utilities/MDreader.C','src/utilities/inverseDistanceWeigthing.C','src/utilities/microSphere.C','src/utilities/NDspline.C','src/utilities/ND_Interpolation_Functions.C','src/distributions/distributionNDBase.C','src/distributions/distributionNDNormal.C','src/distributions/distributionFunctions.C','src/distributions/DistributionContainer.C','src/distributions/distribution_1D.C','src/distributions/randomClass.C','src/distributions/distributionNDCartesianSpline.C'],include_dirs=include_dirs,swig_opts=swig_opts,extra_compile_args=extra_compile_args), - Extension('_interpolationND'+ext,['crow_modules/interpolationND'+ext+'.i','src/utilities/ND_Interpolation_Functions.C','src/utilities/NDspline.C','src/utilities/microSphere.C','src/utilities/inverseDistanceWeigthing.C','src/utilities/MDreader.C','src/distributions/randomClass.C'],include_dirs=include_dirs,swig_opts=swig_opts,extra_compile_args=extra_compile_args)], + ext_modules=[Extension('_distribution1D'+ext,['crow_modules/distribution1D'+ext+'.i','src/distributions/distribution.cxx','src/utilities/MDreader.cxx','src/utilities/inverseDistanceWeigthing.cxx','src/utilities/microSphere.cxx','src/utilities/NDspline.cxx','src/utilities/ND_Interpolation_Functions.cxx','src/distributions/distributionNDBase.cxx','src/distributions/distributionNDNormal.cxx','src/distributions/distributionFunctions.cxx','src/distributions/DistributionContainer.cxx','src/distributions/distribution_1D.cxx','src/distributions/randomClass.cxx','src/distributions/distributionNDCartesianSpline.cxx'],include_dirs=include_dirs,swig_opts=swig_opts,extra_compile_args=extra_compile_args), + Extension('_interpolationND'+ext,['crow_modules/interpolationND'+ext+'.i','src/utilities/ND_Interpolation_Functions.cxx','src/utilities/NDspline.cxx','src/utilities/microSphere.cxx','src/utilities/inverseDistanceWeigthing.cxx','src/utilities/MDreader.cxx','src/distributions/randomClass.cxx'],include_dirs=include_dirs,swig_opts=swig_opts,extra_compile_args=extra_compile_args)], py_modules=['crow_modules.distribution1D'+ext,'crow_modules.interpolationND'+ext], ) diff --git a/doc/user_guide/writeTex.py b/doc/user_guide/writeTex.py index 25f4678a0a..0f9bfcd6a2 100644 --- a/doc/user_guide/writeTex.py +++ b/doc/user_guide/writeTex.py @@ -15,8 +15,8 @@ import sys, os, time import xml.etree.ElementTree as ET #load XML navigation tools -sys.path.append(os.path.join(os.getcwd(),'..','..','framework','utils')) -import xmlUtils +sys.path.append(os.path.join(os.getcwd(),'..','..','framework')) +from utils import xmlUtils def getNode(fname,nodepath): """ diff --git a/framework/CodeInterfaces/MELCOR/melgenInterface.py b/framework/CodeInterfaces/MELCOR/melgenInterface.py index d5e0648026..6e518df454 100644 --- a/framework/CodeInterfaces/MELCOR/melgenInterface.py +++ b/framework/CodeInterfaces/MELCOR/melgenInterface.py @@ -20,7 +20,6 @@ warnings.simplefilter('default',DeprecationWarning) import os -from __builtin__ import any as bAny import GenericParser from utils import utils from CodeInterfaceBaseClass import CodeInterfaceBase @@ -116,7 +115,7 @@ def checkForOutputFailure(self,output,workingDir): return failure readLines = outputToRead.readlines() for goodMsg in errorWord: - if bAny(goodMsg in x for x in readLines): + if any(goodMsg in x for x in readLines): failure = False break return failure diff --git a/framework/CodeInterfaces/RELAP5/Relap5Interface.py b/framework/CodeInterfaces/RELAP5/Relap5Interface.py index cb16cf5b82..ded7eb5a79 100644 --- a/framework/CodeInterfaces/RELAP5/Relap5Interface.py +++ b/framework/CodeInterfaces/RELAP5/Relap5Interface.py @@ -25,7 +25,6 @@ import relapdata import shutil import re -from __builtin__ import any as bAny from CodeInterfaceBaseClass import CodeInterfaceBase class Relap5(CodeInterfaceBase): @@ -107,7 +106,6 @@ def checkForOutputFailure(self,output,workingDir): @ In, workingDir, string, current working dir @ Out, failure, bool, True if the job is failed, False otherwise """ - from __builtin__ import any as bAny failure = True errorWord = ["Transient terminated by end of time step cards","Transient terminated by trip"] try: @@ -116,7 +114,7 @@ def checkForOutputFailure(self,output,workingDir): return failure readLines = outputToRead.readlines() for goodMsg in errorWord: - if bAny(goodMsg in x for x in readLines): + if any(goodMsg in x for x in readLines): failure = False break return failure diff --git a/framework/CustomCommandExecuter.py b/framework/CustomCommandExecuter.py index 1c08a292f5..26e0298915 100644 --- a/framework/CustomCommandExecuter.py +++ b/framework/CustomCommandExecuter.py @@ -43,4 +43,4 @@ def execCommand(commandString,self=None,object=None): @ In, object, instance, optional, object instance @ Out, None """ - execCommandReturn(commandString,self,object) + exec(commandString) diff --git a/framework/DataObjects/Data.py b/framework/DataObjects/Data.py index fe4aabf249..6f5b783dec 100644 --- a/framework/DataObjects/Data.py +++ b/framework/DataObjects/Data.py @@ -33,6 +33,7 @@ import ast import copy import numpy as np +import itertools from scipy import spatial import xml.etree.ElementTree as ET #External Modules End-------------------------------------------------------------------------------- @@ -639,7 +640,7 @@ def getParam(self,typeVar,keyword,nodeId=None,serialize=False): else: if typeVar.lower() in ['input','inputs','unstructuredInput']: returnDict = {} - if keyword in self._dataContainer['inputs'].keys() + self._dataContainer['unstructuredInputs'].keys(): + if keyword in itertools.chain(self._dataContainer['inputs'].keys(),self._dataContainer['unstructuredInputs'].keys()): returnDict[keyword] = {} if self.type == 'HistorySet': for key in self._dataContainer['inputs'][keyword].keys(): diff --git a/framework/DataObjects/PointSet.py b/framework/DataObjects/PointSet.py index 43d325010c..262ef03569 100644 --- a/framework/DataObjects/PointSet.py +++ b/framework/DataObjects/PointSet.py @@ -157,7 +157,7 @@ def _updateSpecializedInputValue(self,name,value,options=None): #self._dataContainer['inputs'][name] = c1darray(values=np.atleast_1d(np.atleast_1d(value)[-1])) if not acceptArrayRealizations else c1darray(values=np.atleast_1d(np.atleast_1d(value))) self.addNodeInTreeMode(tsnode,options) else: - if name in self._dataContainer['inputs'].keys()+self._dataContainer['unstructuredInputs'].keys(): + if name in itertools.chain(self._dataContainer['inputs'].keys(),self._dataContainer['unstructuredInputs'].keys()): #popped = self._dataContainer['inputs'].pop(name) if not unstructuredInput: self._dataContainer['inputs'][name].append(np.atleast_1d(np.ravel(value)[-1])) diff --git a/framework/PostProcessors/BasicStatistics.py b/framework/PostProcessors/BasicStatistics.py index 664b320f82..5a6916011a 100644 --- a/framework/PostProcessors/BasicStatistics.py +++ b/framework/PostProcessors/BasicStatistics.py @@ -748,7 +748,7 @@ def startMetric(metric): metric = 'samples' startMetric(metric) for targetP in needed[metric]: - calculations[metric][targetP] = len(input['targets'].values()[0]) + calculations[metric][targetP] = len(utils.first(input['targets'].values())) # # expected value # diff --git a/framework/Samplers/Sampler.py b/framework/Samplers/Sampler.py index 111c0da6df..6ac547ac43 100644 --- a/framework/Samplers/Sampler.py +++ b/framework/Samplers/Sampler.py @@ -295,7 +295,7 @@ def _readMoreXMLbase(self,xmlNode): self.raiseAnError(IOError,'The following are duplicated variables indices listed in the manifestVariablesIndex: ' + str(dups)) listElement = self.distributions2variablesMapping[dist] for var in listElement: - self.variables2distributionsMapping[var.keys()[0]]['totDim'] = maxDim #reset the totDim to reflect the totDim of original input space + self.variables2distributionsMapping[utils.first(var.keys())]['totDim'] = maxDim #reset the totDim to reflect the totDim of original input space tempListElement = {k.strip():v for x in listElement for ks,v in x.items() for k in list(ks.strip().split(','))} listIndex = [] for var in listLatentElement: diff --git a/framework/SupervisedLearning.py b/framework/SupervisedLearning.py index 53dc44fb1e..18b0c3f812 100644 --- a/framework/SupervisedLearning.py +++ b/framework/SupervisedLearning.py @@ -22,11 +22,16 @@ #for future compatibility with Python 3-------------------------------------------------------------- from __future__ import division, print_function, unicode_literals, absolute_import import warnings -from numpy import average -from crow_modules.distribution1Dpy2 import CDF warnings.simplefilter('default',DeprecationWarning) #End compatibility block for Python 3---------------------------------------------------------------- +from numpy import average +import sys +if sys.version_info.major > 2: + from crow_modules.distribution1Dpy3 import CDF +else: + from crow_modules.distribution1Dpy2 import CDF + #External Modules------------------------------------------------------------------------------------ from sklearn import linear_model from sklearn import svm diff --git a/framework/utils/TreeStructure.py b/framework/utils/TreeStructure.py index cc51218e59..091504521b 100644 --- a/framework/utils/TreeStructure.py +++ b/framework/utils/TreeStructure.py @@ -66,7 +66,8 @@ def parse(inFile,dType=None): raise InputParsingError('Unrecognized file type for:',inFile,' | Expected .xml, .i, .in, or .inp') if dType.lower()=='xml': #try: - xmltree = ET.parse(inFile,parser=XMLCommentParser()) #parser is defined below, under XMLCommentParser + parser = ET.XMLParser(target=CommentedTreeBuilder()) + xmltree = ET.parse(inFile,parser=parser) #parser is defined below, under XMLCommentParser tree = xmlToInputTree(xmltree) #except Exception as e: # print('ERROR: Input parsing error!') @@ -321,32 +322,32 @@ def addChild(gNode,tNode,depth=0): ########### # PARSERS # ########### -class XMLCommentParser(ET.XMLTreeBuilder): +## Extracted from: https://stackoverflow.com/questions/33573807/faithfully-preserve-comments-in-parsed-xml-python-2-7 +## As mentioned on this post, we could also potentially use lxml to handle this +## automatically without the need for a special class +class CommentedTreeBuilder(ET.TreeBuilder): """ - An XML parser that expands on the default to preserves comments + A class for preserving comments faithfully when parsing XML """ - def __init__(self): + def __init__(self, *args, **kwargs): """ - Constructor. - @ In, None - @ Out, None + The constructor that passes arguments to the + xml.etree.ElementTree.TreeBuilder class. + See the relevant documentation for the arguments accepted by the + base class's __init__ function. """ - ET.XMLTreeBuilder.__init__(self) - self._parser.CommentHandler = self.handleComment + super(CommentedTreeBuilder, self).__init__(*args, **kwargs) - def handleComment(self,data): + def comment(self, data): """ - Constructor. - @ In, data, string object to parse into comment - @ Out, None + A function for appropriately surrounding data with the necessary + markers to establish it as a comment. + @ In, data, string, the text that needs to be wrapped in a comment. + @ Out, None """ - self._target.start(ET.Comment,{}) - self._target.data(data) - self._target.end(ET.Comment) - -#set up a parser for this module -parser = XMLCommentParser() - + self.start(ET.Comment, {}) + self.data(data) + self.end(ET.Comment) ######### # NODES # diff --git a/framework/utils/__init__.py b/framework/utils/__init__.py index 319d87a13c..6543e775c4 100644 --- a/framework/utils/__init__.py +++ b/framework/utils/__init__.py @@ -15,6 +15,7 @@ Created on September 16, 2015 @author: maljdp """ +from __future__ import absolute_import # This file is necessary so that the sub-modules understand the correct hierarchy # of things. Once everything is in sub-modules we can possibly do some things # with RAVEN in its entirety as a module, but for now this file can remain diff --git a/framework/utils/utils.py b/framework/utils/utils.py index 5db81acfed..b7c7e65804 100644 --- a/framework/utils/utils.py +++ b/framework/utils/utils.py @@ -385,6 +385,14 @@ def toBytes(s): else: return s +def isString(s): + """ + Method to figure out if a variable is a string. + @ In, s, object, variable to be tested. + @ Out, isString, bool, true if variable is a str or unicode. + """ + return type(s).__name__ in ['unicode','str'] + def toBytesIterative(s): """ Method aimed to convert all the string-compatible content of diff --git a/framework/utils/xmlUtils.py b/framework/utils/xmlUtils.py index 3bc4e88e16..219d0489e7 100644 --- a/framework/utils/xmlUtils.py +++ b/framework/utils/xmlUtils.py @@ -25,6 +25,7 @@ import xml.dom.minidom as pxml import re import os +from .utils import isString #define type checking def isComment(node): @@ -206,16 +207,16 @@ def fixXmlText(msg): @ Out, msg, string, fixed string """ #if not a string, pass it back through - if not isinstance(msg,basestring): + if not isString(msg): return msg #otherwise, replace illegal characters with "?" # from http://boodebr.org/main/python/all-about-python-and-unicode#UNI_XML RE_XML_ILLEGAL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \ u'|' + \ u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \ - (unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff), - unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff), - unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff)) + ('\ud800','\udbff','\udc00','\udfff', + '\ud800','\udbff','\udc00','\udfff', + '\ud800','\udbff','\udc00','\udfff') msg = re.sub(RE_XML_ILLEGAL, "?", msg) return msg @@ -226,7 +227,7 @@ def fixXmlTag(msg): @ Out, msg, string, fixed string """ #if not a string, pass it back through - if not isinstance(msg,basestring): + if not isString(msg): return msg #define some presets letters = u'([a-zA-Z])' diff --git a/setup3.py b/setup3.py index be6dd60208..833c9b6019 100644 --- a/setup3.py +++ b/setup3.py @@ -13,6 +13,22 @@ # limitations under the License. from distutils.core import setup, Extension from distutils.command.build import build +import os + +# Replicating the methods used in the RAVEN Makefile to find CROW_DIR, +# If the Makefile changes to be more robust, so should this +# We should be doing a search for CROW, I would think, we should not force a +# directory structure +CURR_DIR = os.path.dirname(os.path.realpath(__file__)) +HERD_TRUNK_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),'..') +CROW_SUBMODULE = os.path.join(CURR_DIR,'crow') +if os.path.isfile(os.path.join(CROW_SUBMODULE,'Makefile')): + CROW_DIR = CROW_SUBMODULE +else: + CROW_DIR = os.path.join(HERD_TRUNK_DIR,'crow') + +BOOST_INCLUDE_DIR = os.path.join(CROW_DIR,'contrib','include') +RAVEN_INCLUDE_DIR = os.path.join('include','contrib') # We need a custom build order in order to ensure that amsc.py is available # before we try to copy it to the target location @@ -22,15 +38,16 @@ class CustomBuild(build): ('build_clib', build.has_c_libraries), ('build_scripts', build.has_scripts)] -include_dirs=['include/contrib'] -swig_opts=['-c++','-Iinclude/contrib','-py3'] +include_dirs=[RAVEN_INCLUDE_DIR,BOOST_INCLUDE_DIR] +swig_opts=['-c++','-I'+RAVEN_INCLUDE_DIR, '-I'+BOOST_INCLUDE_DIR,'-py3'] +extra_compile_args=['-std=c++11'] setup(name='amsc', version='0.0', description='A library for computing the Approximate Morse-Smale Complex (AMSC)', ext_modules=[Extension('_amsc',['src/contrib/amsc.i', 'src/contrib/UnionFind.cpp', 'src/contrib/AMSC.cpp'], - include_dirs=include_dirs, swig_opts=swig_opts)], + include_dirs=include_dirs, swig_opts=swig_opts,extra_compile_args=extra_compile_args)], package_dir={'':'src/contrib'}, py_modules=['amsc'], cmdclass={'build': CustomBuild}) diff --git a/tests/framework/utils/testParse.py b/tests/framework/utils/testParse.py index c47beb8ba3..e05f5343fc 100644 --- a/tests/framework/utils/testParse.py +++ b/tests/framework/utils/testParse.py @@ -11,6 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +#for future compatibility with Python 3-------------------------------------------------------------- +from __future__ import division, print_function, unicode_literals, absolute_import +import warnings +warnings.simplefilter('default',DeprecationWarning) +#End compatibility block for Python 3---------------------------------------------------------------- + import os,sys import xml.etree.ElementTree as ET @@ -18,6 +25,7 @@ sys.path.append(frameworkDir) import utils.TreeStructure as TS +from utils.utils import toString results = {'failed':0,'passed':0} @@ -30,16 +38,16 @@ def checkSameFile(a,b): while True: i+=1 try: - al = genA.next() + al = next(genA) except StopIteration: try: - genB.next() #see if B is done + next(genB) #see if B is done return False,msg + ['file '+str(b)+' has more lines thani '+str(a)] except StopIteration: #both are done return same,msg try: - bl = genB.next() + bl = next(genB) except StopIteration: return False,msg + ['file '+str(a)+' has more lines than '+str(b)] if al.rstrip('\n\r') != bl.rstrip('\n\r'): @@ -50,56 +58,56 @@ def checkSameFile(a,b): #first test XML to XML -print 'Testing XML to XML ...' +print('Testing XML to XML ...') tree = TS.parse(open(os.path.join('parse','example_xml.xml'),'r')) strTree = TS.tostring(tree) fname = os.path.join('parse','fromXmltoXML.xml') -open(fname,'w').write(strTree) +open(fname,'w').write(toString(strTree)) same,msg = checkSameFile(open(fname,'r'),open(os.path.join('gold',fname),'r')) if same: results['passed']+=1 - print ' ... passed!' + print(' ... passed!') else: results['failed']+=1 - print ' ... failures in XML to XML:' - print ' ',msg[0] + print(' ... failures in XML to XML:') + print(' ',msg[0]) getpot = open(os.path.join('parse','example_getpot.i'),'r') gtree = TS.parse(getpot,dType='GetPot') #third test GetPot to XML -print 'Testing GetPot to XML ...' +print('Testing GetPot to XML ...') strTree = TS.tostring(gtree) fname = os.path.join('parse','fromGetpotToXml.xml') -open(fname,'w').write(strTree) +open(fname,'w').write(toString(strTree)) same,msg = checkSameFile(open(fname,'r'),open(os.path.join('gold',fname),'r')) if same: results['passed']+=1 - print ' ... passed!' + print(' ... passed!') else: results['failed']+=1 - print ' ... failures in GetPot to XML:' - print ' ',msg[0] + print(' ... failures in GetPot to XML:') + print(' ',msg[0]) #finally test XML to GetPot -print 'Testing XML to GetPot ...' +print('Testing XML to GetPot ...') strTree = tree.printGetPot() fname = os.path.join('parse','fromXmltoGetpot.i') open(fname,'w').write(strTree) same,msg = checkSameFile(open(fname,'r'),open(os.path.join('gold',fname),'r')) if same: results['passed']+=1 - print ' ... passed!' + print(' ... passed!') else: results['failed']+=1 - print ' ... failures in GetPot to XML:' - print ' ',msg[0] + print(' ... failures in GetPot to XML:') + print(' ',msg[0]) #second test Getpot to GetPot -print 'Testing GetPot to GetPot ...' +print('Testing GetPot to GetPot ...') getpot = open(os.path.join('parse','example_getpot.i'),'r') gtree = TS.parse(getpot,dType='GetPot') strTree = gtree.printGetPot() @@ -108,11 +116,11 @@ def checkSameFile(a,b): same,msg = checkSameFile(open(fname,'r'),open(os.path.join('gold',fname),'r')) if same: results['passed']+=1 - print ' ... passed!' + print(' ... passed!') else: results['failed']+=1 - print ' ... failures in GetPot to Getpot:' - print ' ',msg[0] + print(' ... failures in GetPot to Getpot:') + print(' ',msg[0]) @@ -121,5 +129,5 @@ def checkSameFile(a,b): -print 'Results:', list('%s: %i' %(k,v) for k,v in results.items()) +print('Results:', list('%s: %i' %(k,v) for k,v in results.items())) sys.exit(results['failed'])