Merge commit 'fe45b8a5b122d1ce740a55dd51281268dd026c3d'

doc/user_manual/existing_interfaces.tex

@@ -1111,8 +1111,36 @@ \subsubsection{Models}
     ...
 </Simulation>
 \end{lstlisting}
-RAVEN works best with Comma-Separated Value (CSV) files.  Therefore, the default .mat output type needs to be converted to .csv output.
-The Dymola interface will automatically convert the .mat output to human-readable forms, i.e., .csv output, through its implementation of the finalizeCodeOutput function.
+RAVEN works best with Comma-Separated Value (CSV) files.  Therefore, the default 
+.mat output type needs to be converted to .csv output.
+The Dymola interface will automatically convert the .mat output to human-readable 
+forms, i.e., .csv output, through its implementation of the finalizeCodeOutput function.
+\\In order to speed up the reading and conversion of the .mat file, the user can specify
+the list of variables (in addition to the Time variable) that need to be imported and 
+converted into a csv file minimizing 
+the IO memory usage as much as possible. Within the \xmlNode{Code} the following 
+XML 
+node (in addition ot the \xmlNode{executable} one) can be inputted:
+
+\begin{itemize}
+   \item \xmlNode{outputVariablesToLoad}, \xmlDesc{space separated list, optional 
+   parameter}, a space separated list of variables that need be exported from the .mat 
+   file (in addition to the Time variable). \default{all the variables in the .mat file}.
+\end{itemize}
+For example:
+\begin{lstlisting}[style=XML]
+<Simulation>
+    ...
+  <Models>
+    <Code name="BouncingBall" subType = "Dymola">
+      <executable>dymosim.exe</executable>
+      <outputVariablesToLoad>var1 var2 var3</outputVariablesToLoad>
+    </Code>
+  </Models>
+    ...
+</Simulation>
+\end{lstlisting}
+
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%% MESH GENERATION COUPLED INTERFACES %%%%%%%

framework/CodeInterfaces/Dymola/DymolaInterface.py

@@ -117,6 +117,20 @@ def __init__(self):
       @ In, None
       @ Out, None
     """
+    self.variablesToLoad = [] # the variables that should be loaded from the mat file (by default, all of them)
+
+  def _readMoreXML(self,xmlNode):
+    """
+      Function to read the portion of the xml input that belongs to this specialized class and initialize
+      some members based on inputs. This can be overloaded in specialize code interface in order to
+      read specific flags.
+      Only one option is possible. You can choose here, if multi-deck mode is activated, from which deck you want to load the results
+      @ In, xmlNode, xml.etree.ElementTree.Element, Xml element node
+      @ Out, None.
+    """
+    child = xmlNode.find("outputVariablesToLoad")
+    if child is not None:
+      self.variablesToLoad = [var.strip() for var in child.text.split()]
 
   #  Generate the command to run Dymola. The form of the command is:
   #
@@ -315,22 +329,23 @@ def finalizeCodeOutput(self, command, output, workingDir):
       @ In, workingDir, string, current working dir
       @ Out, output, string, optional, present in case the root of the output file gets changed in this method.
     """
-    self._vars = {}
-    self._blocks = []
-    self._namesData1 = []
-    self._namesData2 = []
-    self._timeSeriesData1 = []
-    self._timeSeriesData2 = []
+    _vars = {}
+    _blocks = []
+    _namesData1 = []
+    _namesData2 = []
+    _timeSeriesData1 = []
+    _timeSeriesData2 = []
 
     # Load the output file (.mat file) that have been generated by running Dymola executable and
     #   store the data in this file to variable 'mat'.
     matSourceFileName = os.path.join(workingDir, output)
     matSourceFileName += '.mat'
     ###################################################################
     #FIXME: LOADMAT HAS A DIFFERENT BEHAVIOR IN SCIPY VERSION >= 0.18 #
-    if int(scipy.__version__.split(".")[1])>17:
-      warnings.warn("SCIPY version >0.17.xx has a different behavior in reading .mat files!")
+    #if int(scipy.__version__.split(".")[1])>17:
+    #  warnings.warn("SCIPY version >0.17.xx has a different behavior in reading .mat files!")
     mat = scipy.io.loadmat(matSourceFileName, chars_as_strings=False)
+
     ###################################################################
 
     # Define the functions that extract strings from the matrix:
@@ -360,11 +375,11 @@ def finalizeCodeOutput(self, command, output, workingDir):
         c = abs(x)-1  # column (reduced)
         s = sign(x)   # sign
         if c:
-          self._vars[names[i]] = (descr[i], d, c, float(s))
-          if not d in self._blocks:
-            self._blocks.append(d)
+          _vars[names[i]] = (descr[i], d, c, float(s))
+          if not d in _blocks:
+            _blocks.append(d)
         else:
-          self._absc = (names[i], descr[i])
+          _absc = (names[i], descr[i])
 
       # Extract the trajectory for the variable 'Time' and store the data in the variable 'timeSteps'.
       timeSteps = mat['data_2'][0]
@@ -376,39 +391,43 @@ def finalizeCodeOutput(self, command, output, workingDir):
       timeStepsArray = numpy.array([timeSteps])
 
       # Extract the names and output points of all variables and store them in the variables:
-      #  - self._namesData1: Names of parameters
-      #  - self._namesData2: Names of the variables that are not parameters
-      #  - self._timeSeriesData1: Trajectories (time series data) of 'self._namesData1'
-      #  - self._timeSeriesData2: Trajectories (time series data) of 'self._namesData2'
-      for (k,v) in self._vars.items():
-        dataValue = mat['data_%d' % (v[1])][v[2]]
-        if v[3] < 0:
-          dataValue = dataValue * -1.0
-        if v[1] == 1:
-          self._namesData1.append(k)
-          self._timeSeriesData1.append(dataValue)
-        elif v[1] == 2:
-          self._namesData2.append(k)
-          self._timeSeriesData2.append(dataValue)
-        else:
-          raise Exception('File structure not supported!')
-      timeSeriesData1 = numpy.array(self._timeSeriesData1)
-      timeSeriesData2 = numpy.array(self._timeSeriesData2)
+      #  - _namesData1: Names of parameters
+      #  - _namesData2: Names of the variables that are not parameters
+      #  - _timeSeriesData1: Trajectories (time series data) of '_namesData1'
+      #  - _timeSeriesData2: Trajectories (time series data) of '_namesData2'
+      for (k,v) in _vars.items():
+        readIt = True
+        if len(self.variablesToLoad) > 0 and k not in self.variablesToLoad:
+          readIt = False
+        if readIt:
+          dataValue = mat['data_%d' % (v[1])][v[2]]
+          if v[3] < 0:
+            dataValue = dataValue * -1.0
+          if v[1] == 1:
+            _namesData1.append(k)
+            _timeSeriesData1.append(dataValue)
+          elif v[1] == 2:
+            _namesData2.append(k)
+            _timeSeriesData2.append(dataValue)
+          else:
+            raise Exception('File structure not supported!')
+      timeSeriesData1 = numpy.array(_timeSeriesData1)
+      timeSeriesData2 = numpy.array(_timeSeriesData2)
 
       # The csv writer places quotes arround variables that contain a ',' in the name, i.e.
       # a, "b,c", d would represent 3 variables 1) a 2) b,c 3) d. The csv reader in RAVEN does not
       # suport this convention.
       # => replace ',' in variable names with '@', i.e.
       # a, "b,c", d will become a, b@c, d
-      for mylist in [self._namesData1, self._namesData2]:
+      for mylist in [_namesData1, _namesData2]:
         for i in range(len(mylist)):
           if ',' in mylist[i]:
             mylist[i]  = mylist[i].replace(',', '@')
 
       # Recombine the names of the variables and insert the variable 'Time'.
-      # Order of the variable names should be 'Time', self._namesData1, self._namesData2.
+      # Order of the variable names should be 'Time', _namesData1, _namesData2.
       # Also, convert the type of the resulting variable from 'list' to '2-d array'.
-      varNames = numpy.array([[self._absc[0]] + self._namesData1 + self._namesData2])
+      varNames = numpy.array([[_absc[0]] + _namesData1 + _namesData2])
 
       # Compute the number of parameters.
       sizeParams = timeSeriesData1.shape[0]
@@ -435,4 +454,15 @@ def finalizeCodeOutput(self, command, output, workingDir):
         resultsWriter.writerows(varTrajectories)
     else:
       raise Exception('File structure not supported!')
+    #release memory
+    del _vars
+    del _blocks
+    del _namesData1
+    del _namesData2
+    del _timeSeriesData1
+    del _timeSeriesData2
+    del _absc
+    del Data1Array
+    del timeSeriesData1
+    del timeSeriesData2
     return os.path.splitext(destFileName)[0]   # Return the name without the .csv on it as RAVEN will add it later.

tests/framework/CodeInterfaceTests/DMInterfaceTestLoadSomeVars/dsin.txt

@@ -0,0 +1,195 @@
+#1
+char Aclass(3,24)
+Adymosim
+1.4
+Modelica experiment file
+
+
+#    Experiment parameters
+double experiment(7,1)
+       0                   # StartTime    Time at which integration starts
+                           #              (and linearization and trimming time)
+      10                   # StopTime     Time at which integration stops
+ 1.0000000000000001E-001   # Increment    Communication step size, if > 0
+       0                   # nInterval    Number of communication intervals, if > 0
+ 1.0000000000000000E-004   # Tolerance    Relative precision of signals for
+                           #              simulation, linearization and trimming
+       0                   # MaxFixedStep Maximum step size of fixed step size
+                           #              integrators, if > 0.0
+       8                   # Algorithm    Integration algorithm as integer (1...28)
+                           #
+                           #             | model|       |        | dense | state |
+                           # Algorithm   | typ  | stiff | order  | output| event |
+                           # ------------+------+-------+--------+-------+-------+
+                           #  1 | deabm  |  ode |   no  |  1-12  |  yes  |   no  |
+                           #  2 | lsode1 |  ode |   no  |  1-12  |  yes  |   no  |
+                           #  3 | lsode2 |  ode |  yes  |  1-5   |  yes  |   no  |
+                           #  4 | lsodar |  ode |  both |1-12,1-5|  yes  |  yes  |
+                           #  5 | dopri5 |  ode |   no  |   5    |   no  |   no  |
+                           #  6 | dopri8 |  ode |   no  |   8    |   no  |   no  |
+                           #  7 | grk4t  |  ode |  yes  |   4    |   no  |   no  |
+                           #  8 | dassl  |  dae |  yes  |  1-5   |  yes  |  yes  |
+                           #  9 | odassl | hdae |  yes  |  1-5   |  yes  |  yes  |
+                           # 10 | mexx   | hdae |   no  |  2-24  |   no  |   no  |
+                           # 11 | euler  |  ode |   no  |   1    |   no  |  yes  |
+                           # 12 | rkfix2 |  ode |   no  |   2    |   no  |  yes  |
+                           # 13 | rkfix3 |  ode |   no  |   3    |   no  |  yes  |
+                           # 14 | rkfix4 |  ode |   no  |   4    |   no  |  yes  |
+                           #>=14| others |  ode |yes/no |  2-5   |   yes |  yes  |
+                           # ---+--------+------+-------+--------+-------+-------+
+                           # euler and rkfix have fixed stepsize.
+
+
+#    Method tuning parameters
+double method(27,1)
+       1                   # grid     type of communication time grid, defined by
+                           #          = 1: equidistant points ("Increment/nInterval")
+                           #          = 2: vector of grid points ("tgrid")
+                           #          = 3: variable step integrator (automatically)
+                           #          = 4: model (call of "increment" in Dymola, e.g.
+                           #                      incr=Time > 2 then 0 else 0.1
+                           #                      dummy=increment(incr))
+                           #          grid = 1,3 is stopped by "StopTime"
+                           #          grid = 2   is stopped by "tgrid(last)"
+                           #          grid = 4   runs forever (stopped by model)
+       1                   # nt       Use every NT time instant, if grid = 3
+       3                   # dense    1/2/3 restart/step/interpolate GRID points
+       1                   # evgrid   0/1 do not/save event points in comm. time grid
+       1                   # evu      0/1 U-discontinuity does not/trigger events
+       0                   # evuord       U-discontinuity order to consider (0,1,...)
+       0                   # error    0/1/2 One message/warning/error messages
+       0                   # jac      0/1 Compute jacobian numerically/by BLOCKJ
+       0                   # xd0c     0/1 Compute/set XD0
+       0                   # f3       0/1 Ignore/use F3 of HDAE (= index 1)
+       0                   # f4       0/1 Ignore/use F4 of HDAE (= index 2)
+       0                   # f5       0/1 Ignore/use F5 of HDAE (= invar.)
+       0                   # debug    flags for debug information (1<<0 uses pdebug) 
+     100                   # pdebug       priority of debug information (1...100)
+       0                   # fmax     Maximum number of evaluations of BLOCKF, if > 0
+       0                   # ordmax   Maximum allowed integration order, if > 0
+       0                   # hmax     Maximum absolute stepsize, if > 0
+       0                   # hmin     Minimum absolute stepsize, if > 0 (use with care!)
+       0                   # h0       Stepsize to be attempted on first step, if > 0
+ 2.0000000000000000E-014   # teps     Bound to check, if 2 equal time instants
+ 1.0000000000000000E-010   # eveps    Hysteresis epsilon at event points
+      20                   # eviter   Maximum number of event iterations
+ 9.9999999999999995E-007   # delaym   Minimum time increment in delay buffers
+       1                   # fexcep   0/1 floating exception crashes/stops dymosim
+       1                   # tscale   clock-time = tscale*simulation-time, if grid = 5
+                           #          > 1: simulation too slow
+                           #          = 1: simulation-time = real-time
+                           #          < 1: simulation too fast
+       1                   # shared   (not used)
+    2473                   # memkey   (not used)
+
+
+#    Output parameters
+int settings(13,1)
+ 0                         # lprec    0/1 do not/store result data in double
+ 1                         # lx       0/1 do not/store x  (state variables)
+ 1                         # lxd      0/1 do not/store xd (derivative of states)
+ 1                         # lu       0/1 do not/store u  (input     signals)
+ 1                         # ly       0/1 do not/store y  (output    signals)
+ 0                         # lz       0/1 do not/store z  (indicator signals)
+ 1                         # lw       0/1 do not/store w  (auxiliary signals)
+ 1                         # la       0/1 do not/store a  (alias     signals)
+ 0                         # lperf    0/1 do not/store performance indicators
+ 0                         # levent   0/1 do not/store event point
+ 1                         # lres     0/1 do not/store results on result file
+ 0                         # lshare   0/1 do not/store info data for shared memory on dsshare.txt
+ 1                         # lform    0/1 ASCII/Matlab-binary storage format of results
+                           #              (for simulation/linearization; not for trimming)
+
+
+#    Names of initial variables
+char initialName(12,6)
+e
+g
+hstart
+vstart
+h
+der(h)
+v
+der(v)
+flying
+impact
+v_new
+foo
+
+double initialValue(12,6)
+ -1 6.9999999999999996E-001  0  0  1  280   # e
+ -1 9.8100000000000005E+000  0  0  1  280   # g
+ -1      10                  0  0  1  280   # hstart
+ -1       0                  0  0  1  280   # vstart
+ -1       0                  0  0  2  272   # h
+  0       0                  0  0  3  256   # der(h)
+ -1       0                  0  0  2  272   # v
+  0       0                  0  0  3  256   # der(v)
+  0       1                  0  0  6  329   # flying
+  0       0                  0  0  6  329   # impact
+  0       0                  0  0  6  328   # v_new
+  0       0                  0  0  6  322   # foo
+
+
+# Matrix with 6 columns defining the initial value calculation
+# (columns 5 and 6 are not utilized for the calculation but are
+#  reported by dymosim via dymosim -i for user convenience):
+#
+# column 1: Type of initial value
+#           = -2: special case: for continuing simulation (column 2 = value)
+#           = -1: fixed value (column 2 = fixed value)
+#           =  0: free value, i.e., no restriction (column 2 = initial value)
+#           >  0: desired value (column 1 = weight for optimization
+#                                column 2 = desired value)
+#                 use weight=1, since automatic scaling usually
+#                 leads to equally weighted terms
+# column 2: fixed, free or desired value according to column 1.
+# column 3: Minimum value (ignored, if Minimum >= Maximum).
+# column 4: Maximum value (ignored, if Minimum >= Maximum).
+#           Minimum and maximum restrict the search range in initial
+#           value calculation. They might also be used for scaling.
+# column 5: Category of variable.
+#           = 1: parameter.
+#           = 2: state.
+#           = 3: state derivative.
+#           = 4: output.
+#           = 5: input.
+#           = 6: auxiliary variable.
+# column 6: Data type of variable and flags according to dsBaseType
+#  <value>&3= 0: real.
+#  <value>&3= 1: boolean.
+#  <value>&3= 2: integer.
+#
+# Initial values are calculated according to the following procedure:
+#
+# - If parameters, states and inputs are FIXED, and other variables
+#   are FREE, no special action takes place (default setting).
+#
+# - If there are only FIXED and FREE variables and the number of
+#   FREE parameters, states and inputs is IDENTICAL to the number of
+#   FIXED state derivatives, outputs and auxiliary variables, a non-linear
+#   equation is solved to determine a consistent set of initial conditions.
+#
+# - In all other cases the following optimization problem is solved:
+#      min( sum( weight(i)*( (value(i) - DESIRED(i))/scale(i) )^2 ) )
+#   under the constraint that the differential equation is fulfilled
+#   at the initial time. In most cases weight(i)=1 is sufficient, due
+#   to the automatic scaling (if DESIRED(i) is not close to zero,
+#   scale(i) = DESIRED(i). Otherwise, the scaling is based on the
+#   nominal value (and maybe minimum and maximum values given in 
+#   column 3 and 4). If these values are zero, scale(i)=1 is used).
+#
+char initialDescription(12,42)
+coefficient of restitution
+gravity acceleration
+height of ball at time zero
+velocity of ball at time zero
+height of ball
+der(height of ball)
+velocity of ball
+der(velocity of ball)
+true, if ball is flying [:#(type=Boolean)]
+[:#(type=Boolean)]
+
+[:#(type=Integer)]
+