diff --git a/FWCore/Reflection/python/ClassesDefXmlUtils.py b/FWCore/Reflection/python/ClassesDefXmlUtils.py new file mode 100644 index 0000000000000..5cbdf71c035c9 --- /dev/null +++ b/FWCore/Reflection/python/ClassesDefXmlUtils.py @@ -0,0 +1,143 @@ +class XmlParser(object): + """Parses a classes_def.xml file looking for class declarations that contain + ClassVersion attributes. Once found looks for sub-elements named 'version' + which contain the ClassVersion to checksum mappings. + """ + + #The following are constants used to describe what data is kept + # in which index in the 'classes' member data + originalNameIndex=0 + classVersionIndex=1 + versionsToChecksumIndex = 2 + + def __init__(self, filename, includeNonVersionedClasses=False, normalizeClassNames=True): + self._file = filename + self.classes = dict() + self._presentClass = None + self._presentClassForVersion = None + self._includeNonVersionedClasses = includeNonVersionedClasses + self._normalizeClassNames = normalizeClassNames + self.readClassesDefXML() + def readClassesDefXML(self): + import xml.parsers.expat + p = xml.parsers.expat.ParserCreate() + p.StartElementHandler = self.start_element + p.EndElementHandler = self.end_element + f = open(self._file) + # Replace any occurence of <>& in the attribute values by the xml parameter + rxml, nxml = f.read(), '' + q1,q2 = 0,0 + for c in rxml : + if (q1 or q2) and c == '<' : nxml += '<' + elif (q1 or q2) and c == '>' : nxml += '>' + # elif (q1 or q2) and c == '&' : nxml += '&' + else : nxml += c + if c == '"' : q1 = not q1 + if c == "'" : q2 = not q2 + try : p.Parse(nxml) + except xml.parsers.expat.ExpatError as e : + print ('--->> edmCheckClassVersion: ERROR: parsing selection file ',self._file) + print ('--->> edmCheckClassVersion: ERROR: Error is:', e) + raise + f.close() + def start_element(self,name,attrs): + if name in ('class','struct'): + if 'name' in attrs: + self._presentClass=attrs['name'] + normalizedName = self.genNName(attrs['name']) + if 'ClassVersion' in attrs: + self.classes[normalizedName]=[attrs['name'],int(attrs['ClassVersion']),[]] + self._presentClassForVersion=normalizedName + elif self._includeNonVersionedClasses: + # skip transient data products + if not ('persistent' in attrs and attrs['persistent'] == "false"): + self.classes[normalizedName]=[attrs['name'],-1,[]] + else: + raise RuntimeError(f"There is an element '{name}' without 'name' attribute.") + if name == 'version': + if self._presentClassForVersion is None: + raise RuntimeError(f"Class element for type '{self._presentClass}' contains a 'version' element, but 'ClassVersion' attribute is missing from the 'class' element") + try: + classVersion = int(attrs['ClassVersion']) + except KeyError: + raise RuntimeError(f"Version element for type '{self._presentClass}' is missing 'ClassVersion' attribute") + try: + checksum = int(attrs['checksum']) + except KeyError: + raise RuntimeError(f"Version element for type '{self._presentClass}' is missing 'checksum' attribute") + self.classes[self._presentClassForVersion][XmlParser.versionsToChecksumIndex].append([classVersion, checksum]) + pass + def end_element(self,name): + if name in ('class','struct'): + self._presentClass = None + self._presentClassForVersion = None + def genNName(self, name ): + if not self._normalizeClassNames: + return name + n_name = " ".join(name.split()) + for e in [ ['long long unsigned int', 'unsigned long long'], + ['long long int', 'long long'], + ['unsigned short int', 'unsigned short'], + ['short unsigned int', 'unsigned short'], + ['short int', 'short'], + ['long unsigned int', 'unsigned long'], + ['unsigned long int', 'unsigned long'], + ['long int', 'long'], + ['std::string', 'std::basic_string']] : + n_name = n_name.replace(e[0],e[1]) + n_name = n_name.replace(' ','') + return n_name + +def initROOT(library): + #Need to not have ROOT load .rootlogon.(C|py) since it can cause interference. + import ROOT + ROOT.PyConfig.DisableRootLogon = True + + #Keep ROOT from trying to use X11 + ROOT.gROOT.SetBatch(True) + ROOT.gROOT.ProcessLine(".autodict") + if library is not None: + if ROOT.gSystem.Load(library) < 0 : + raise RuntimeError("failed to load library '"+library+"'") + +def initCheckClass(): + """Must be called before checkClass()""" + import ROOT + ROOT.gROOT.ProcessLine("class checkclass {public: int f(char const* name) {TClass* cl = TClass::GetClass(name); bool b = false; cl->GetCheckSum(b); return (int)b;} };") + ROOT.gROOT.ProcessLine("checkclass checkTheClass;") + + +#The following are error codes returned from checkClass +noError = 0 +errorRootDoesNotMatchClassDef =1 +errorMustUpdateClassVersion=2 +errorMustAddChecksum=3 + +def checkClass(name,version,versionsToChecksums): + import ROOT + c = ROOT.TClass.GetClass(name) + if not c: + raise RuntimeError("failed to load dictionary for class '"+name+"'") + temp = "checkTheClass.f(" + '"' + name + '"' + ");" + retval = ROOT.gROOT.ProcessLine(temp) + if retval == 0 : + raise RuntimeError("TClass::GetCheckSum: Failed to load dictionary for base class. See previous Error message") + classChecksum = c.GetCheckSum() + classVersion = c.GetClassVersion() + + #does this version match what is in the file? + if version != classVersion: + return (errorRootDoesNotMatchClassDef,classVersion,classChecksum) + + #is the version already in our list? + found = False + + for v,cs in versionsToChecksums: + if v == version: + found = True + if classChecksum != cs: + return (errorMustUpdateClassVersion,classVersion,classChecksum) + break + if not found and classVersion != 0: + return (errorMustAddChecksum,classVersion,classChecksum) + return (noError,classVersion,classChecksum) diff --git a/FWCore/Reflection/scripts/edmCheckClassVersion b/FWCore/Reflection/scripts/edmCheckClassVersion new file mode 100755 index 0000000000000..88fbcb58ba1f1 --- /dev/null +++ b/FWCore/Reflection/scripts/edmCheckClassVersion @@ -0,0 +1,144 @@ +#! /usr/bin/env python3 + +import sys +import FWCore.Reflection.ClassesDefXmlUtils as ClassesDefUtils + +# recursively check the base classes for a class pointer +# as building the streamer will crash if base classes are +# incomplete +def verifyBaseClasses(c) : + missingBase = 0 + + # check that all bases are loaded + bases = c.GetListOfBases() + if not bases : + print ("Incomplete class ", c.GetName()) + return 1 + + for b in bases : + bc = b.GetClassPointer() + if bc : + missingBase += verifyBaseClasses(bc) + else : + print ("Incomplete base class for ", c.GetName(), ": ", b.GetName()) + missingBase += 1 + + return missingBase + +def checkDictionaries(name): + c = ROOT.TClass.GetClass(name) + if not c: + raise RuntimeError("failed to load dictionary for class '"+name+"'") + + missingDict = verifyBaseClasses(c) + if missingDict == 0 : + si = c.GetStreamerInfo() + if si : + ts = si.GetElements() + for telem in ts : + clm = telem.GetClassPointer() + if clm and not clm.IsLoaded() : + print ("Missing dictionary for ", telem.GetName(), " type ", clm.GetName()) + missingDict += 1 + else : + print ("No streamer info for ", c.GetName()) + missingDict += 1 + + return missingDict + +#Setup the options +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +oparser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) +oparser.add_argument("-d","--check_dictionaries", dest="checkdict",action="store_true",default=False, + help="check that all required dictionaries are loaded") +oparser.add_argument("-l","--lib", dest="library", type=str, + help="specify the library to load. If not set classes are found using the PluginManager") +oparser.add_argument("-x","--xml_file", dest="xmlfile",default="./classes_def.xml", type=str, + help="the classes_def.xml file to read") +oparser.add_argument("-g","--generate_new",dest="generate", action="store_true",default=False, + help="instead of issuing errors, generate a new classes_def.xml file.") + +options=oparser.parse_args() + +ClassesDefUtils.initROOT(options.library) +if options.library is None and options.checkdict: + print ("Dictionary checks require a specific library") + +missingDict = 0 + +ClassesDefUtils.initCheckClass() + +try: + p = ClassesDefUtils.XmlParser(options.xmlfile) +except RuntimeError as e: + print(f"Parsing {options.xmlfile} failed: {e}") + sys.exit(1) +foundErrors = dict() +for name,info in p.classes.items(): + errorCode,rootClassVersion,classChecksum = ClassesDefUtils.checkClass(name,info[ClassesDefUtils.XmlParser.classVersionIndex],info[ClassesDefUtils.XmlParser.versionsToChecksumIndex]) + if errorCode != ClassesDefUtils.noError: + foundErrors[name]=(errorCode,classChecksum,rootClassVersion) + if options.checkdict : + missingDict += checkDictionaries(name) + +foundRootDoesNotMatchError = False +originalToNormalizedNames = dict() +for name,retValues in foundErrors.items(): + origName = p.classes[name][ClassesDefUtils.XmlParser.originalNameIndex] + originalToNormalizedNames[origName]=name + code = retValues[0] + classVersion = p.classes[name][ClassesDefUtils.XmlParser.classVersionIndex] + classChecksum = retValues[1] + rootClassVersion = retValues[2] + if code == ClassesDefUtils.errorRootDoesNotMatchClassDef: + foundRootDoesNotMatchError=True + print ("error: for class '"+name+"' ROOT says the ClassVersion is "+str(rootClassVersion)+" but classes_def.xml says it is "+str(classVersion)+". Are you sure everything compiled correctly?") + elif code == ClassesDefUtils.errorMustUpdateClassVersion and not options.generate: + print ("error: class '"+name+"' has a different checksum for ClassVersion "+str(classVersion)+". Increment ClassVersion to "+str(classVersion+1)+" and assign it to checksum "+str(classChecksum)) + elif not options.generate: + print ("error:class '"+name+"' needs to include the following as part of its 'class' declaration") + print (' ') + + +if options.generate and not foundRootDoesNotMatchError and not missingDict: + f = open(options.xmlfile) + outFile = open('classes_def.xml.generated','w') + out = '' + for l in f.readlines(): + newLine = l + if -1 != l.find('')) + if code == ClassesDefUtils.errorMustUpdateClassVersion: + classVersion += 1 + parts = splitArgs[:] + indexToClassVersion = 0 + for pt in parts: + indexToClassVersion +=1 + if -1 != pt.find('ClassVersion'): + break + parts[indexToClassVersion]=str(classVersion) + newLine = '"'.join(parts) + + if hasNoSubElements: + newLine = newLine.replace('/','') + out +=newLine + newLine =' '*indent+' \n' + if hasNoSubElements: + out += newLine + newLine=' '*indent+'\n' + out +=newLine + + outFile.writelines(out) + +if (len(foundErrors)>0 and not options.generate) or (options.generate and foundRootDoesNotMatchError) or missingDict: + import sys + sys.exit(1) + diff --git a/FWCore/Reflection/scripts/edmDumpClassVersion b/FWCore/Reflection/scripts/edmDumpClassVersion new file mode 100755 index 0000000000000..a92011a342312 --- /dev/null +++ b/FWCore/Reflection/scripts/edmDumpClassVersion @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +import sys +import json +import argparse + +import FWCore.Reflection.ClassesDefXmlUtils as ClassesDefUtils + +def main(args): + ClassesDefUtils.initROOT(args.library) + + ClassesDefUtils.initCheckClass() + try: + p = ClassesDefUtils.XmlParser(args.xmlfile, includeNonVersionedClasses=True, normalizeClassNames=False) + except RuntimeError as e: + print(f"Parsing {args.xmlfile} failed: {e}") + sys.exit(1) + + out = {} + for name, info in p.classes.items(): + try: + (error, version, checksum) = ClassesDefUtils.checkClass(name, 0, {}) + except RuntimeError as e: + print(f"Ignoring class {name} as could not get its version and checksum, because: {e}") + continue + out[name] = dict( + version = version, + checksum = checksum + ) + out_js = json.dumps(out, sort_keys=True, indent=1) + if args.output is None: + print(out_js) + else: + with open(args.output, "w") as f: + f.write(out_js) + return 0 + +if __name__ == "__main__": + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Extracts class versions and checksums, in JSON format, for all non-transient clases defined in a given classes_def.xml file") + parser.add_argument("-l","--lib", dest="library", type=str, + help="specify the library to load. If not set classes are found using the PluginManager") + parser.add_argument("-x","--xml_file", dest="xmlfile",default="./classes_def.xml", type=str, + help="the classes_def.xml file to read") + parser.add_argument("-o", "--output", type=str, default=None, + help="File to save the output. If no file is specified, the JSON document is printed in stdout") + + args = parser.parse_args() + sys.exit(main(args)) diff --git a/FWCore/Reflection/test/BuildFile.xml b/FWCore/Reflection/test/BuildFile.xml new file mode 100644 index 0000000000000..10e0d1ae71daf --- /dev/null +++ b/FWCore/Reflection/test/BuildFile.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/FWCore/Reflection/test/dumpClassVersion_reference.json b/FWCore/Reflection/test/dumpClassVersion_reference.json new file mode 100644 index 0000000000000..2032fd1a08d78 --- /dev/null +++ b/FWCore/Reflection/test/dumpClassVersion_reference.json @@ -0,0 +1,10 @@ +{ + "edm::Wrapper": { + "checksum": 536952283, + "version": 4 + }, + "edmtest::reflection::IntObject": { + "checksum": 427917710, + "version": 3 + } +} \ No newline at end of file diff --git a/FWCore/Reflection/test/run_checkClassVersion.sh b/FWCore/Reflection/test/run_checkClassVersion.sh new file mode 100755 index 0000000000000..43bd19707b8e9 --- /dev/null +++ b/FWCore/Reflection/test/run_checkClassVersion.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +function die { echo Failure $1: status $2 ; exit $2 ; } + +XMLPATH=${SCRAM_TEST_PATH}/stubs +LIBFILE=${LOCALTOP}/lib/${SCRAM_ARCH}/libFWCoreReflectionTestObjects.so + +edmCheckClassVersion -l ${LIBFILE} -x ${XMLPATH}/classes_def.xml || die "edmCheckClassVersion failed" $? + +function runFailure { + edmCheckClassVersion -l ${LIBFILE} -x ${XMLPATH}/$1 > log.txt && die "edmCheckClassVersion for $1 did not fail" 1 + grep -q "$2" log.txt + RET=$? + if [ "$RET" != "0" ]; then + echo "edmCheckClassVersion for $1 did not contain '$2', log is below" + cat log.txt + exit 1 + fi +} + +runFailure test_def_nameMissing.xml "There is an element 'class' without 'name' attribute" +runFailure test_def_ClassVersionMissingInClass.xml "Class element for type 'edmtest::reflection::IntObject' contains a 'version' element, but 'ClassVersion' attribute is missing from the 'class' element" +runFailure test_def_ClassVersionMissingInVersion.xml "Version element for type 'edmtest::reflection::IntObject' is missing 'ClassVersion' attribute" +runFailure test_def_checksumMissingInVersion.xml "Version element for type 'edmtest::reflection::IntObject' is missing 'checksum' attribute" diff --git a/FWCore/Reflection/test/run_dumpClassVersion.sh b/FWCore/Reflection/test/run_dumpClassVersion.sh new file mode 100755 index 0000000000000..824050547577b --- /dev/null +++ b/FWCore/Reflection/test/run_dumpClassVersion.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +function die { echo Failure $1: status $2 ; exit $2 ; } + +XMLPATH=${SCRAM_TEST_PATH}/stubs +LIBFILE=${LOCALTOP}/lib/${SCRAM_ARCH}/libFWCoreReflectionTestObjects.so + +edmDumpClassVersion -l ${LIBFILE} -x ${XMLPATH}/classes_def.xml -o dump.json || die "edmDumpClassVersion failed" $? +diff -u ${SCRAM_TEST_PATH}/dumpClassVersion_reference.json dump.json || die "Unexpected class version dump" $? + +function runFailure() { + edmDumpClassVersion -l ${LIBFILE} -x ${XMLPATH}/$1 > log.txt && die "edmDumpClassVersion for $1 did not fail" 1 + grep -q "$2" log.txt + RET=$? + if [ "$RET" != "0" ]; then + echo "edmDumpClassVersion for $1 did not contain '$2', log is below" + cat log.txt + exit 1 + fi +} + +runFailure test_def_nameMissing.xml "There is an element 'class' without 'name' attribute" +runFailure test_def_ClassVersionMissingInClass.xml "Class element for type 'edmtest::reflection::IntObject' contains a 'version' element, but 'ClassVersion' attribute is missing from the 'class' element" +runFailure test_def_ClassVersionMissingInVersion.xml "Version element for type 'edmtest::reflection::IntObject' is missing 'ClassVersion' attribute" +runFailure test_def_checksumMissingInVersion.xml "Version element for type 'edmtest::reflection::IntObject' is missing 'checksum' attribute" diff --git a/FWCore/Reflection/test/stubs/TestObjects.cc b/FWCore/Reflection/test/stubs/TestObjects.cc new file mode 100644 index 0000000000000..b5a6333f7d510 --- /dev/null +++ b/FWCore/Reflection/test/stubs/TestObjects.cc @@ -0,0 +1,5 @@ +#include "TestObjects.h" + +namespace edmtest::reflection { + IntObject::IntObject() = default; +} diff --git a/FWCore/Reflection/test/stubs/TestObjects.h b/FWCore/Reflection/test/stubs/TestObjects.h new file mode 100644 index 0000000000000..0ebbb9ec79fb4 --- /dev/null +++ b/FWCore/Reflection/test/stubs/TestObjects.h @@ -0,0 +1,17 @@ +#ifndef FWCore_Reflection_test_TestObjects_h +#define FWCore_Reflection_test_TestObjects_h + +namespace edmtest::reflection { + class IntObject { + public: + IntObject(); + IntObject(int v) : value_(v) {} + + int get() const { return value_; } + + private: + int value_ = 0; + }; +} // namespace edmtest::reflection + +#endif diff --git a/FWCore/Reflection/test/stubs/classes.h b/FWCore/Reflection/test/stubs/classes.h new file mode 100644 index 0000000000000..975b0eaca2005 --- /dev/null +++ b/FWCore/Reflection/test/stubs/classes.h @@ -0,0 +1,2 @@ +#include "DataFormats/Common/interface/Wrapper.h" +#include "FWCore/Reflection/test/stubs/TestObjects.h" diff --git a/FWCore/Reflection/test/stubs/classes_def.xml b/FWCore/Reflection/test/stubs/classes_def.xml new file mode 100644 index 0000000000000..c8a2b6538439b --- /dev/null +++ b/FWCore/Reflection/test/stubs/classes_def.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/FWCore/Reflection/test/stubs/test_def_ClassVersionMissingInClass.xml b/FWCore/Reflection/test/stubs/test_def_ClassVersionMissingInClass.xml new file mode 100644 index 0000000000000..208aceb2d4399 --- /dev/null +++ b/FWCore/Reflection/test/stubs/test_def_ClassVersionMissingInClass.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/FWCore/Reflection/test/stubs/test_def_ClassVersionMissingInVersion.xml b/FWCore/Reflection/test/stubs/test_def_ClassVersionMissingInVersion.xml new file mode 100644 index 0000000000000..6024d072151da --- /dev/null +++ b/FWCore/Reflection/test/stubs/test_def_ClassVersionMissingInVersion.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/FWCore/Reflection/test/stubs/test_def_checksumMissingInVersion.xml b/FWCore/Reflection/test/stubs/test_def_checksumMissingInVersion.xml new file mode 100644 index 0000000000000..4d536eaf3b3a3 --- /dev/null +++ b/FWCore/Reflection/test/stubs/test_def_checksumMissingInVersion.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/FWCore/Reflection/test/stubs/test_def_nameMissing.xml b/FWCore/Reflection/test/stubs/test_def_nameMissing.xml new file mode 100644 index 0000000000000..3e1ba8ce07f77 --- /dev/null +++ b/FWCore/Reflection/test/stubs/test_def_nameMissing.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/FWCore/Utilities/scripts/edmCheckClassVersion b/FWCore/Utilities/scripts/edmCheckClassVersion deleted file mode 100755 index 8737d180d203c..0000000000000 --- a/FWCore/Utilities/scripts/edmCheckClassVersion +++ /dev/null @@ -1,255 +0,0 @@ -#! /usr/bin/env python3 -from sys import version_info -if version_info[0] > 2: - atol = int -else: - from string import atol - -class XmlParser(object): - """Parses a classes_def.xml file looking for class declarations that contain - ClassVersion attributes. Once found looks for sub-elements named 'version' - which contain the ClassVersion to checksum mappings. - """ - - #The following are constants used to describe what data is kept - # in which index in the 'classes' member data - originalNameIndex=0 - classVersionIndex=1 - versionsToChecksumIndex = 2 - - def __init__(self,filename): - self._file = filename - self.classes = dict() - self._presentClass = None - self.readClassesDefXML() - def readClassesDefXML(self): - import xml.parsers.expat - p = xml.parsers.expat.ParserCreate() - p.StartElementHandler = self.start_element - p.EndElementHandler = self.end_element - f = open(self._file) - # Replace any occurence of <>& in the attribute values by the xml parameter - rxml, nxml = f.read(), '' - q1,q2 = 0,0 - for c in rxml : - if (q1 or q2) and c == '<' : nxml += '<' - elif (q1 or q2) and c == '>' : nxml += '>' - # elif (q1 or q2) and c == '&' : nxml += '&' - else : nxml += c - if c == '"' : q1 = not q1 - if c == "'" : q2 = not q2 - try : p.Parse(nxml) - except xml.parsers.expat.ExpatError as e : - print ('--->> edmCheckClassVersion: ERROR: parsing selection file ',self._file) - print ('--->> edmCheckClassVersion: ERROR: Error is:', e) - raise - f.close() - def start_element(self,name,attrs): - if name in ('class','struct'): - if 'name' in attrs: - if 'ClassVersion' in attrs: - normalizedName = self.genNName(attrs['name']) - self.classes[normalizedName]=[attrs['name'],atol(attrs['ClassVersion']),[]] - self._presentClass=normalizedName - if name == 'version': - self.classes[self._presentClass][XmlParser.versionsToChecksumIndex].append([atol(attrs['ClassVersion']), - atol(attrs['checksum'])]) - pass - def end_element(self,name): - if name in ('class','struct'): - self._presentClass = None - def genNName(self, name ): - n_name = " ".join(name.split()) - for e in [ ['long long unsigned int', 'unsigned long long'], - ['long long int', 'long long'], - ['unsigned short int', 'unsigned short'], - ['short unsigned int', 'unsigned short'], - ['short int', 'short'], - ['long unsigned int', 'unsigned long'], - ['unsigned long int', 'unsigned long'], - ['long int', 'long'], - ['std::string', 'std::basic_string']] : - n_name = n_name.replace(e[0],e[1]) - n_name = n_name.replace(' ','') - return n_name - -# recursively check the base classes for a class pointer -# as building the streamer will crash if base classes are -# incomplete -def verifyBaseClasses(c) : - missingBase = 0 - - # check that all bases are loaded - bases = c.GetListOfBases() - if not bases : - print ("Incomplete class ", c.GetName()) - return 1 - - for b in bases : - bc = b.GetClassPointer() - if bc : - missingBase += verifyBaseClasses(bc) - else : - print ("Incomplete base class for ", c.GetName(), ": ", b.GetName()) - missingBase += 1 - - return missingBase - -def checkDictionaries(name): - c = ROOT.TClass.GetClass(name) - if not c: - raise RuntimeError("failed to load dictionary for class '"+name+"'") - - missingDict = verifyBaseClasses(c) - if missingDict == 0 : - si = c.GetStreamerInfo() - if si : - ts = si.GetElements() - for telem in ts : - clm = telem.GetClassPointer() - if clm and not clm.IsLoaded() : - print ("Missing dictionary for ", telem.GetName(), " type ", clm.GetName()) - missingDict += 1 - else : - print ("No streamer info for ", c.GetName()) - missingDict += 1 - - return missingDict - -#The following are error codes returned from checkClass -noError = 0 -errorRootDoesNotMatchClassDef =1 -errorMustUpdateClassVersion=2 -errorMustAddChecksum=3 - -def checkClass(name,version,versionsToChecksums): - c = ROOT.TClass.GetClass(name) - if not c: - raise RuntimeError("failed to load dictionary for class '"+name+"'") - temp = "checkTheClass.f(" + '"' + name + '"' + ");" - retval = ROOT.gROOT.ProcessLine(temp) - if retval == 0 : - raise RuntimeError("TClass::GetCheckSum: Failed to load dictionary for base class. See previous Error message") - classChecksum = c.GetCheckSum() - classVersion = c.GetClassVersion() - - #does this version match what is in the file? - if version != classVersion: - return (errorRootDoesNotMatchClassDef,classVersion,classChecksum) - - #is the version already in our list? - found = False - - for v,cs in versionsToChecksums: - if v == version: - found = True - if classChecksum != cs: - return (errorMustUpdateClassVersion,classVersion,classChecksum) - break - if not found and classVersion != 0: - return (errorMustAddChecksum,classVersion,classChecksum) - return (noError,classVersion,classChecksum) - -#Setup the options -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -oparser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -oparser.add_argument("-d","--check_dictionaries", dest="checkdict",action="store_true",default=False, - help="check that all required dictionaries are loaded") -oparser.add_argument("-l","--lib", dest="library", type=str, - help="specify the library to load. If not set classes are found using the PluginManager") -oparser.add_argument("-x","--xml_file", dest="xmlfile",default="./classes_def.xml", type=str, - help="the classes_def.xml file to read") -oparser.add_argument("-g","--generate_new",dest="generate", action="store_true",default=False, - help="instead of issuing errors, generate a new classes_def.xml file.") - -options=oparser.parse_args() - -#Need to not have ROOT load .rootlogon.(C|py) since it can cause interference. -import ROOT -ROOT.PyConfig.DisableRootLogon = True - -#Keep ROOT from trying to use X11 -ROOT.gROOT.SetBatch(True) -ROOT.gROOT.ProcessLine(".autodict") -if options.library is None: - if options.checkdict : - print ("Dictionary checks require a specific library") -else: - if ROOT.gSystem.Load(options.library) < 0 : - raise RuntimeError("failed to load library '"+options.library+"'") - -missingDict = 0 - -ROOT.gROOT.ProcessLine("class checkclass {public: int f(char const* name) {TClass* cl = TClass::GetClass(name); bool b = false; cl->GetCheckSum(b); return (int)b;} };") -ROOT.gROOT.ProcessLine("checkclass checkTheClass;") - -p = XmlParser(options.xmlfile) -foundErrors = dict() -for name,info in p.classes.items(): - errorCode,rootClassVersion,classChecksum = checkClass(name,info[XmlParser.classVersionIndex],info[XmlParser.versionsToChecksumIndex]) - if errorCode != noError: - foundErrors[name]=(errorCode,classChecksum,rootClassVersion) - if options.checkdict : - missingDict += checkDictionaries(name) - -foundRootDoesNotMatchError = False -originalToNormalizedNames = dict() -for name,retValues in foundErrors.items(): - origName = p.classes[name][XmlParser.originalNameIndex] - originalToNormalizedNames[origName]=name - code = retValues[0] - classVersion = p.classes[name][XmlParser.classVersionIndex] - classChecksum = retValues[1] - rootClassVersion = retValues[2] - if code == errorRootDoesNotMatchClassDef: - foundRootDoesNotMatchError=True - print ("error: for class '"+name+"' ROOT says the ClassVersion is "+str(rootClassVersion)+" but classes_def.xml says it is "+str(classVersion)+". Are you sure everything compiled correctly?") - elif code == errorMustUpdateClassVersion and not options.generate: - print ("error: class '"+name+"' has a different checksum for ClassVersion "+str(classVersion)+". Increment ClassVersion to "+str(classVersion+1)+" and assign it to checksum "+str(classChecksum)) - elif not options.generate: - print ("error:class '"+name+"' needs to include the following as part of its 'class' declaration") - print (' ') - - -if options.generate and not foundRootDoesNotMatchError and not missingDict: - f = open(options.xmlfile) - outFile = open('classes_def.xml.generated','w') - out = '' - for l in f.readlines(): - newLine = l - if -1 != l.find('')) - if code == errorMustUpdateClassVersion: - classVersion += 1 - parts = splitArgs[:] - indexToClassVersion = 0 - for pt in parts: - indexToClassVersion +=1 - if -1 != pt.find('ClassVersion'): - break - parts[indexToClassVersion]=str(classVersion) - newLine = '"'.join(parts) - - if hasNoSubElements: - newLine = newLine.replace('/','') - out +=newLine - newLine =' '*indent+' \n' - if hasNoSubElements: - out += newLine - newLine=' '*indent+'\n' - out +=newLine - - outFile.writelines(out) - -if (len(foundErrors)>0 and not options.generate) or (options.generate and foundRootDoesNotMatchError) or missingDict: - import sys - sys.exit(1) - diff --git a/Utilities/ReleaseScripts/scripts/checkDictionaryUpdate.py b/Utilities/ReleaseScripts/scripts/checkDictionaryUpdate.py new file mode 100755 index 0000000000000..c6539bbde6398 --- /dev/null +++ b/Utilities/ReleaseScripts/scripts/checkDictionaryUpdate.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import sys +import json +import argparse + +# exit codes +NO_CHANGES = 0 +DATAFORMATS_CHANGED = 40 +POLICY_VIOLATION = 41 + +def policyChecks(document): + """Check policies on dictionary definitions. Return True if checks are fine.""" + # Contents to be added later + return True + +def updatePolicyChecks(reference, update): + """Check policies on dictionary updates. Return True if checks are fine.""" + # Contents to be added later + return True + +def main(args): + with open(args.baseline) as f: + baseline = json.load(f) + + if args.pr is not None: + with open(args.pr) as f: + pr = json.load(f) + pc1 = policyChecks(pr) + if baseline != pr: + pc2 = updatePolicyChecks(baseline, pr) + if not (pc1 and pc2): + return POLICY_VIOLATION + + print("Changes in persistable data formats") + return DATAFORMATS_CHANGED + if not pc1: + return POLICY_VIOLATION + else: + if not policyChecks(baseline): + return POLICY_VIOLATION + + return NO_CHANGES + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=f"Check dictionary policies of the JSON output of edmDumpClassVersion. If one JSON document is given (--baseline; e.g. in IBs), only the dictionary definition policy checks are done. If two JSON documents are given (--baseline and --pr; e.g. in PR tests), the dictionary definition policy checks are done on the --pr document, and, in addition, if any persistable data formats are changed, additional checks are done to ensure the dictionary update is done properly. Exits with {NO_CHANGES} if there are no changes to persistent data formats. Exits with {DATAFORMATS_CHANGED} if persistent data formats are changed, and the update complies with data format policies. Exits with {POLICY_VIOLATION} if some data format policy is violated. Other exit codes (e.g. 1, 2) denote some failure in the script itself.") + + parser.add_argument("--baseline", required=True, type=str, help="JSON file for baseline") + parser.add_argument("--pr", type=str, help="JSON file for baseline+PR") + parser.add_argument("--transientDataFormatPackage", action="store_true", help="The JSON files are for a package that can have only transient data formats") + + args = parser.parse_args() + sys.exit(main(args)) diff --git a/Utilities/ReleaseScripts/test/BuildFile.xml b/Utilities/ReleaseScripts/test/BuildFile.xml index 9c17b56d1c695..cb3a56d1959ad 100644 --- a/Utilities/ReleaseScripts/test/BuildFile.xml +++ b/Utilities/ReleaseScripts/test/BuildFile.xml @@ -11,3 +11,5 @@ + + diff --git a/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_baseline.json b/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_baseline.json new file mode 100644 index 0000000000000..2032fd1a08d78 --- /dev/null +++ b/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_baseline.json @@ -0,0 +1,10 @@ +{ + "edm::Wrapper": { + "checksum": 536952283, + "version": 4 + }, + "edmtest::reflection::IntObject": { + "checksum": 427917710, + "version": 3 + } +} \ No newline at end of file diff --git a/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_newClass.json b/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_newClass.json new file mode 100644 index 0000000000000..0d370acd84448 --- /dev/null +++ b/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_newClass.json @@ -0,0 +1,14 @@ +{ + "edm::Wrapper": { + "checksum": 536952283, + "version": 4 + }, + "edmtest::reflection::IntObject": { + "checksum": 427917710, + "version": 3 + }, + "edmtest::reflection::Another": { + "checksum": 427917711, + "version": 3 + } +} diff --git a/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_removeClass.json b/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_removeClass.json new file mode 100644 index 0000000000000..a3b92aec2d77e --- /dev/null +++ b/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_removeClass.json @@ -0,0 +1,6 @@ +{ + "edmtest::reflection::IntObject": { + "checksum": 427917710, + "version": 3 + } +} diff --git a/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_versionUpdate.json b/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_versionUpdate.json new file mode 100644 index 0000000000000..3b7d96e50163a --- /dev/null +++ b/Utilities/ReleaseScripts/test/checkDictionaryUpdate/dumpClassVersion_versionUpdate.json @@ -0,0 +1,10 @@ +{ + "edm::Wrapper": { + "checksum": 536952283, + "version": 4 + }, + "edmtest::reflection::IntObject": { + "checksum": 3848242946, + "version": 4 + } +} diff --git a/Utilities/ReleaseScripts/test/run_checkDictionaryUpdate.sh b/Utilities/ReleaseScripts/test/run_checkDictionaryUpdate.sh new file mode 100755 index 0000000000000..151465dda1ed0 --- /dev/null +++ b/Utilities/ReleaseScripts/test/run_checkDictionaryUpdate.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# possible exit codes +SUCCESS=0 +FAILURE="fail" +DATAFORMATS_CHANGED=40 +POLICY_VIOLATION=41 + +function die { echo Failure $1: status $2 ; exit $2 ; } +function checkExitCode { + if [ "$1" == "$FAILURE" ]; then + if [[ "$2" = @("$SUCCESS"|"$DATAFORMATS_CHANGES"|"$POLICY_VIOLATIONS") ]]; then + echo "checkDictionaryUpdate.py $3: expected failure exit code, got $2" + exit 1 + fi + elif [ "$1" != "$2" ]; then + echo "checkDictionaryUpdate.py $3: expected exit code $1, got $2" + exit 1 + fi +} + +JSONPATH=${SCRAM_TEST_PATH}/checkDictionaryUpdate + +checkDictionaryUpdate.py --baseline ${JSONPATH}/dumpClassVersion_baseline.json || die "checkDictionaryUpdate.py baseline" $? +checkDictionaryUpdate.py --baseline ${JSONPATH}/dumpClassVersion_baseline.json --pr ${JSONPATH}/dumpClassVersion_baseline.json || die "checkDictionaryUpdate.py baseline baseline" $? + +checkDictionaryUpdate.py +RET=$? +checkExitCode ${FAILURE} $RET "" + +checkDictionaryUpdate.py --baseline ${JSONPATH}/dumpClassVersion_baseline.json --pr ${JSONPATH}/dumpClassVersion_versionUpdate.json +RET=$? +checkExitCode ${DATAFORMATS_CHANGED} $RET "baseline versionUpdate" + +checkDictionaryUpdate.py --baseline ${JSONPATH}/dumpClassVersion_baseline.json --pr ${JSONPATH}/dumpClassVersion_newClass.json +RET=$? +checkExitCode ${DATAFORMATS_CHANGED} $RET "baseline newClass" + +checkDictionaryUpdate.py --baseline ${JSONPATH}/dumpClassVersion_baseline.json --pr ${JSONPATH}/dumpClassVersion_removeClass.json +RET=$? +checkExitCode ${DATAFORMATS_CHANGED} $RET "baseline removeClass"