More intelligent Variable Groups (#803)

* implementation, tests, and conversion script (not run on all tests yet)

* updated tests

* updated docs

* fixed print, failing tests

doc/user_manual/Makefile

@@ -1,6 +1,6 @@
 SRCFILE = raven_user_manual
 # MANUAL_FILES = optimizer.tex rom.tex postprocessor.tex database_data.tex OutStreamSystem.tex sampler.tex existing_interfaces.tex ProbabilityDistributions.tex  step.tex functions.tex ravenStructure.tex Summary.tex introduction.tex raven_user_manual.tex model.tex runInfo.tex libraries.tex DataMining.tex HowToRun.tex metrics.tex
-MANUAL_FILES = optimizer.tex rom.tex postprocessor.tex database_data.tex OutStreamSystem.tex sampler.tex \
+MANUAL_FILES = optimizer.tex rom.tex postprocessor.tex database_data.tex OutStreamSystem.tex sampler.tex variablegroups.tex \
 existing_interfaces.tex ProbabilityDistributions.tex  step.tex functions.tex ravenStructure.tex Summary.tex \
 introduction.tex raven_user_manual.tex model.tex runInfo.tex libraries.tex DataMining.tex HowToRun.tex metrics.tex \
 Installation/clone.tex Installation/conda.tex Installation/linux.tex Installation/macosx.tex Installation/main.tex \

doc/user_manual/variablegroups.tex

@@ -8,8 +8,9 @@ \section{VariableGroups}
 Each entry in the \xmlNode{VariableGroups} block has a distinct name and list of each constituent variable in
 the group.
 %
-Alternatively, set operations can be used to construct variable groups from other variable groups.  In this case,
-the dependent groups and the base group on which operations should be performed must be listed.  The following types of
+Additionally, set operations can be used to construct variable groups from other variable groups, by listing
+them in node text along with the operation to perform.
+The following types of
 set operations are included in RAVEN:
 \begin{itemize}
   \item \texttt{+}, Union, the combination of all variables in the \xmlString{base} set and listed set,
@@ -18,8 +19,9 @@ \section{VariableGroups}
   \item \texttt{\%}, Symmetric Difference, the variables in only either the \xmlString{base} or listed set,
     but not both.
 \end{itemize}
-Multiple operations can be performed by separating them with commas in the text of the group node.  In the
-event the listed set is a single variable, it will be treated like a set with a single entry.
+Multiple set operations can be performed by separating them with commas in the text of the group node, whether
+they be variable groups or single variables. In the event a circular dependency loop is detected, an error will be
+raised. VariableGroups are evaluated in the order of entries listed in their node text.
 
 When using the variable groups in a node, they can be listed alone or as part of a comma-separated list.  The
 variable group name will only be substituted in the text of nodes, not attributes or tags.
@@ -31,13 +33,6 @@ \section{VariableGroups}
   \item \xmlAttr{name}, \xmlDesc{required string attribute}, user-defined name
   of the group. This is the identifier that will be used elsewhere in the RAVEN input.
   %
-  \item \xmlAttr{dependencies}, \xmlDesc{optional comma-separated string attribute}, the other variable groups
-    on which this group is dependent for construction.  If listed, all entries in the text of this node must
-    be proceeded by one of the set operators above.  Defaults to an empty string.
-  \item \xmlAttr{base}, \xmlDesc{optional string attribute}, the starting set for constructing a dependent
-    variable group.  This attribute is required if any dependencies are listed.  Set operations are performed
-    by performing the chosen operation on the variable group listed in this attribute along with the listed
-    variable group.  No default.
 \end{itemize}
 \vspace{-5mm}
 
@@ -48,15 +43,15 @@ \section{VariableGroups}
 <Simulation>
   ...
   <VariableGroups>
-    <Group name="x_odd"  >x1,x3,x5</Group>
-    <Group name="x_even" >x2,x4,x6</Group>
-    <Group name="x_first">x1,x2,x3</Group>
-    <Group name="y_group">y1,y2</Group>
-    <Group name="add_remove" dependencies="x_first"         base="x_first">-x1,+ x4,+x5</Group>
-    <Group name="union"      dependencies="x_odd,x_even"    base="x_odd">+x_even</Group>
-    <Group name="complement" dependencies="x_odd,x_first"   base="x_odd">-x_first</Group>
-    <Group name="intersect"  dependencies="x_even,x_first"  base="x_even">^x_first</Group>
-    <Group name="sym_diff"   dependencies="x_odd,x_first"   base="x_odd">% x_first</Group>
+    <Group name="x_odd"     >x1,x3,x5</Group>
+    <Group name="x_even"    >x2,x4,x6</Group>
+    <Group name="x_first"   >x1,x2,x3</Group>
+    <Group name="y_group"   >y1,y2</Group>
+    <Group name="add_remove">x_first,-x1,+ x4,+x5</Group>
+    <Group name="union"     >x_odd,+x_even</Group>
+    <Group name="complement">x_odd,-x_first</Group>
+    <Group name="intersect" >x_even,^x_first</Group>
+    <Group name="sym_diff"  >x_odd,% x_first</Group>
   </VariableGroups>
   ...
   <DataObjects>

framework/VariableGroups.py

@@ -33,6 +33,10 @@
 import BaseClasses
 #Internal Modules End--------------------------------------------------------------------------------
 
+#
+#
+#
+#
 class VariableGroup(BaseClasses.BaseType):
   """
     Allows grouping of variables for ease of access
@@ -45,113 +49,69 @@ def __init__(self):
     """
     BaseClasses.BaseType.__init__(self)
     self.printTag       = 'VariableGroup'
-    self._dependents    = []             #name of groups this group's construction is dependent on
-    self._base          = None           #if dependent, the name of base group to start from
-    self._list          = []             #text from node
     self.variables      = []             #list of variable names
     self.initialized    = False          #true when initialized
 
-  def _readMoreXML(self,node):
+  def readXML(self, node, messageHandler, varGroups):
     """
       reads XML for more information
       @ In, node, xml.etree.ElementTree.Element, xml element to read data from
+      @ In, varGroups, dict, other variable groups including ones this depends on (if any)
       @ Out, None
     """
+    self.messageHandler = messageHandler
     #establish the name
     if 'name' not in node.attrib.keys():
       self.raiseAnError(IOError,'VariableGroups require a "name" attribute!')
     self.name = node.attrib['name']
-    #dependents
-    deps = node.attrib.get('dependencies',None)
-    if deps is not None and len(deps)>0:
-      if 'base' not in node.attrib.keys():
-        self.raiseAnError(IOError,'VariableGroups with dependencies require a "base" group to start from!')
-      self._base = node.attrib.get('base')
-      self._dependents = list(g.strip() for g in deps.split(','))
-    self._list = node.text.split(',')
+    # loop through variables and expand list
+    for dep in [s.strip() for s in node.text.split(',')]:
+      # get operator if provided
+      operator = '+'
+      if dep[0] in '+-^%':
+        operator = dep[0]
+        dep = dep[1:].strip()
+      # expand variables if a group name is given
+      if dep in varGroups:
+        dep = varGroups[dep].getVars()
+      else:
+        dep = [dep]
 
-  def initialize(self,varGroups):
-    """
-      Establish variable set.
-      @ In, varGroups, list, VariableGroup classes
-      @ Out, None
-    """
-    if len(self._dependents)==0:
-      self.variables = list(l.strip() for l in self._list) #set(l.strip() for l in self._list) #don't use sets, since they destroy order
-    else:
-      #get base
-      base = None
-      for group in varGroups:
-        if group.name==self._base:
-          base = group
-          break
-      if base is None:
-        self.raiseAnError(IOError,'Base %s not found among variable groups!' %self._base)
-      #get dependencies
-      deps=OrderedDict()
-      for depName in self._dependents:
-        dep = None
-        for group in varGroups:
-          if group.name==depName:
-            dep = group
-            break
-        if dep is None:
-          self.raiseAnError(IOError,'Dependent %s not found among variable groups!' %depName)
-        deps[depName] = dep
-      #get base set
-      baseVars = set(base.getVars())
-      #do modifiers to base
-      modifiers = list(m.strip() for m in self._list)
-      orderOps = [] #order of operations that occurred, just var names and dep lists
-      for mod in modifiers:
-        #remove internal whitespace
-        mod = mod.replace(' ','')
-        #get operator and varname
-        op = mod[0]
-        varName = mod[1:]
-        if op not in ['+','-','^','%']:
-          self.raiseAnError(IOError,'Unrecognized or missing dependency operator:',op,varName)
-        #if varName is a single variable, make it a set so it behaves like the rest
-        if varName not in deps.keys():
-          modSet = [varName]
-        else:
-          modSet = deps[varName].getVars()
-        orderOps.append(modSet[:])
-        modSet = set(modSet)
-        if   op == '+':
-          baseVars.update(modSet)
-        elif op == '-':
-          baseVars.difference_update(modSet)
-        elif op == '^':
-          baseVars.intersection_update(modSet)
-        elif op == '%':
-          baseVars.symmetric_difference_update(modSet)
-      #sort variable list into self.variables
-      #  -> first, sort through top-level vars
-      for var in base.getVars():
-        if var in baseVars:
-          self.variables.append(var)
-          baseVars.remove(var)
-      #  -> then, sort through deps/operations in order
-      for mod in orderOps:
-        for var in mod:
-          if var in baseVars:
-            self.variables.append(var)
-            baseVars.remove(var)
-      #  -> baseVars better be empty now!
-      if len(baseVars) > 0:
-        self.raiseAWarning('    End vars    :',self.variables)
-        self.raiseAWarning('    End BaseVars:',baseVars)
-        self.raiseAnError(RuntimeError, 'Not all variableGroup entries were accounted for!  The operations were not performed correctly')
-    self.initialized=True
+      # apply operators
+      toRemove = []
+      ## union
+      if operator == '+':
+        for d in dep:
+          if d not in self.variables:
+            self.variables.append(d)
+      ## difference
+      elif operator == '-':
+        for d in dep:
+          try:
+            self.variables.remove(d)
+          except ValueError:
+            self.raiseADebug('Was asked to remove "{}" from variable group "{}", but it is not present! Ignoring ...'
+                               .format(d,self.name))
+      ## intersection
+      elif operator == '^':
+        for v in self.variables:
+          if v not in dep:
+            toRemove.append(v)
+      ## symmetric difference
+      elif operator == '%':
+        for v in self.variables:
+          if v in dep:
+            toRemove.append(v)
+        for d in dep:
+          if d not in self.variables:
+            self.variables.append(d)
+      ## cleanup
+      for v in toRemove:
+        self.variables.remove(v)
+
+    # finished
+    self.raiseADebug('Variable group "{}" includes:'.format(self.name),self.getVarsString())
 
-  def getDependencies(self):
-    """
-      Returns list object of strings containing variable group names
-      @ In, None
-      @ Out, _dependents, list(str), list of variable group names
-    """
-    return self._dependents[:]
 
   def getVars(self):
     """
@@ -172,4 +132,4 @@ def getVarsString(self,delim=','):
 #
 #
 #
-# end
+#

framework/utils/xmlUtils.py

@@ -26,6 +26,7 @@
 import re
 import os
 from .utils import isString
+from .graphStructure import graphObject
 import VariableGroups
 
 #define type checking
@@ -371,27 +372,36 @@ def readVariableGroups(xmlNode,messageHandler,caller):
     @ In, caller, MessageHandler.MessageUser instance, entity calling this method (needs to inherit from MessageHandler.MessageUser)
     @ Out, varGroups, dict, dictionary of variable groups (names to the variable lists to replace the names)
   """
-  varGroups = {}
+  # first find all the names
+  names = [node.attrib['name'] for node in xmlNode]
+
+  # find dependencies
+  deps = {}
+  nodes = {}
+  initials = []
   for child in xmlNode:
-    varGroup = VariableGroups.VariableGroup()
-    varGroup.readXML(child,messageHandler)
-    varGroups[varGroup.name]=varGroup
-  # initialize variable groups
-  while any(not vg.initialized for vg in varGroups.values()):
-    numInit = 0 #new vargroups initialized this pass
-    for vg in varGroups.values():
-      if vg.initialized:
-        continue
-      try:
-        deps = list(varGroups[dp] for dp in vg.getDependencies())
-      except KeyError as e:
-        caller.raiseAnError(IOError,'Dependency %s listed but not found in varGroups!' %e)
-      if all(varGroups[dp].initialized for dp in vg.getDependencies()):
-        vg.initialize(varGroups.values())
-        numInit+=1
-    if numInit == 0:
-      caller.raiseAWarning('variable group status:')
-      for name,vg in varGroups.items():
-        caller.raiseAWarning('   ',name,':',vg.initialized)
-      caller.raiseAnError(RuntimeError,'There was an infinite loop building variable groups!')
+    name = child.attrib['name']
+    nodes[name] = child
+    needs = [s.strip().strip('-+^%') for s in child.text.split(',')]
+    for n in needs:
+      if n not in deps and n not in names:
+        deps[n] = []
+    deps[name] = needs
+    if len(deps[name]) == 0:
+      initials.append(name)
+  graph = graphObject(deps)
+  # sanity checking
+  if graph.isALoop():
+    caller.raiseAnError(IOError,'VariableGroups have circular dependency! Order:',' -> '.join(graph.createSingleListOfVertices(alls)))
+  # ordered list (least dependencies first)
+  hierarchy = list(reversed(graph.createSingleListOfVertices(graph.findAllUniquePaths(initials))))
+
+  # build entities
+  varGroups = {}
+  for name in hierarchy:
+    if len(deps[name]):
+      varGroup = VariableGroups.VariableGroup()
+      varGroup.readXML(nodes[name], messageHandler, varGroups)
+      varGroups[name] = varGroup
+
   return varGroups

scripts/conversionScripts/vargroups_nobases.py

@@ -0,0 +1,40 @@
+# Copyright 2017 Battelle Energy Alliance, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import xml.etree.ElementTree as ET
+import xml.dom.minidom as pxml
+import os
+
+def convert(tree,fileName=None):
+  """
+    Removes the "base" and "dependencies" from VariableGroups and places them in the text.
+    @ In, tree, xml.etree.ElementTree.ElementTree object, the contents of a RAVEN input file
+    @ In, fileName, the name for the raven input file
+    @Out, tree, xml.etree.ElementTree.ElementTree object, the modified RAVEN input file
+  """
+  simulation = tree.getroot()
+  groupsNode = simulation.find('VariableGroups')
+  if groupsNode is not None:
+    for node in groupsNode:
+      if 'base' in node.attrib:
+        node.text = node.attrib['base'] + ',' + node.text
+        del node.attrib['base']
+        del node.attrib['dependencies']
+      elif 'dependencies' in node.attrib:
+        del node.attrib['dependencies']
+  return tree
+
+if __name__=='__main__':
+  import convert_utils
+  import sys
+  convert_utils.standardMain(sys.argv,convert)

tests/framework/VariableGroups/ExternalNodes/vargroups.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" ?>
 <VariableGroups>
-      <Group name="x_odd"                                     >x1,x3,x5</Group>
-      <Group name="x_even"     dependencies=""                >x2,x4,x6</Group>
-      <Group name="y_group"    dependencies=""                >y1,y2</Group>
-      <Group name="union"      dependencies="x_odd,x_even"    base="x_odd">+x_even</Group>
+      <Group name="x_odd"  >x1,x3,x5</Group>
+      <Group name="x_even" >x2,x4,x6</Group>
+      <Group name="y_group">y1,y2</Group>
+      <Group name="union"  >x_odd, x_even</Group>
 </VariableGroups>