global: improvements to ROOT writer, replace test data for YODA 1.7.3

* Improve ROOT writer for case of unordered or non-overlapping bins.
* Demand all bins have 'low' and 'high' before writing ROOT histogram.
* Try converting numbers given as strings to floats for ROOT/YODA.
* Replace full.yoda (using YODA 1.7.3) in test data.
* Refinements to Docker installation instructions in docs.

Signed-off-by: Graeme Watt <[email protected]>

docs/installation.rst

@@ -74,16 +74,25 @@ containing the dependencies such as YODA and ROOT, but not the ``hepdata-convert
 .. code-block:: console
 
     $ docker pull hepdata/hepdata-converter
-    $ docker run -it --rm -v $PWD:$PWD -w $PWD hepdata/hepdata-converter /bin/bash
+    $ docker run --rm -it hepdata/hepdata-converter /bin/bash
 
 The ``hepdata-converter`` package can be installed inside the Docker container:
 
-.. code:: bash
+.. code-block:: console
+
+    root@617be04cbab5:/# pip install --ignore-installed hepdata-converter
+    root@617be04cbab5:/# hepdata-converter -h
+    root@617be04cbab5:/# python -c 'import hepdata_converter'
+
+Note that the Docker container will be automatically removed when it exits (if running with the ``--rm`` option).  The
+Python module or CLI can then be used as described in :doc:`Usage <usage>`.  Input and output files can be moved
+between the local filesystem and the running Docker container using the ``docker cp`` command, for example,
+
+.. code-block:: console
 
-    pip install hepdata-converter
-    hepdata-converter -h
-    python -c 'import hepdata_converter'
+    $ docker cp sample.oldhepdata 617be04cbab5:/
+    root@617be04cbab5:/# hepdata-converter -i oldhepdata sample.oldhepdata SampleYAML
+    $ docker cp 617be04cbab5:/SampleYAML .
 
-The Python module or CLI can then be used as described in :doc:`Usage <usage>` to convert files
-given in the directory given by ``$PWD`` where the Docker container was run.  Note that the Docker
-container will be automatically removed when it exits.
+where the prompt ``$`` indicates a terminal corresponding to the local filesystem and the prompt
+``root@617be04cbab5:/#`` indicates another terminal corresponding to the running Docker container.

hepdata_converter/testsuite/testdata/yoda/full.yoda

@@ -1,25 +1,27 @@
 BEGIN YODA_SCATTER2D_V2 /REF/RIVET_ANALYSIS_NAME/d01-x01-y01
+Variations: [""]
 ErrorBreakdown: {0: {stat: {dn: -3.0, up: 3.3}, sys: {dn: -1.2, up: 1}, 'sys,lumi': {dn: -1, up: 1}}, 1: {stat: {dn: -3.5, up: 3.8}, sys: {dn: -1.5, up: 1.7}, 'sys,lumi': {dn: -1.2, up: 1.2}}, 2: {stat: {dn: -2.9, up: 3.1}, sys: {dn: -1.7, up: 1.7}, 'sys,lumi': {dn: -0.5, up: 0.5}}}
 
 IsRef: 1
 Path: /REF/RIVET_ANALYSIS_NAME/d01-x01-y01
 Title: Table 1
 Type: Scatter2D
 ---
-# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+
+# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+	
 7.000000e+03	0.000000e+00	0.000000e+00	2.540000e+01	3.382307e+00	3.590265e+00
 8.000000e+03	0.000000e+00	0.000000e+00	2.980000e+01	3.992493e+00	4.332436e+00
 9.000000e+03	0.000000e+00	0.000000e+00	1.270000e+01	3.398529e+00	3.570714e+00
 END YODA_SCATTER2D_V2
 BEGIN YODA_SCATTER3D_V2 /REF/RIVET_ANALYSIS_NAME/d02-x01-y01
+Variations: [""]
 ErrorBreakdown: {}
 
 IsRef: 1
 Path: /REF/RIVET_ANALYSIS_NAME/d02-x01-y01
 Title: Table 2
 Type: Scatter3D
 ---
-# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+	 zval	 zerr-	 zerr+
+# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+	 zval	  zerr-	 zerr+	
 1.200000e+02	0.000000e+00	0.000000e+00	4.000000e+02	0.000000e+00	0.000000e+00	2.229000e-03	0.000000e+00	0.000000e+00
 1.200000e+02	0.000000e+00	0.000000e+00	6.000000e+02	0.000000e+00	0.000000e+00	1.190000e-03	0.000000e+00	0.000000e+00
 1.200000e+02	0.000000e+00	0.000000e+00	7.000000e+02	0.000000e+00	0.000000e+00	5.152000e-04	0.000000e+00	0.000000e+00
@@ -52,68 +54,73 @@ Type: Scatter3D
 7.900000e+02	0.000000e+00	0.000000e+00	8.000000e+02	0.000000e+00	0.000000e+00	2.502000e-04	0.000000e+00	0.000000e+00
 END YODA_SCATTER3D_V2
 BEGIN YODA_SCATTER2D_V2 /REF/RIVET_ANALYSIS_NAME/d03-x01-y01
+Variations: [""]
 ErrorBreakdown: {0: {stat: {dn: -1100, up: 1100}, 'sys,background': {dn: -15, up: 15}, 'sys,detector': {dn: -79, up: 79}}, 1: {stat: {dn: -1600, up: 1600}, 'sys,background': {dn: -15, up: 15}, 'sys,detector': {dn: -75, up: 75}}, 2: {stat: {dn: -490, up: 490}, 'sys,background': {dn: -2, up: 2}, 'sys,detector': {dn: -41, up: 41}}, 3: {stat: {dn: -60, up: 60}, 'sys,background': {dn: 0, up: 0}, 'sys,detector': {dn: -2, up: 2}}}
 
 IsRef: 1
 Path: /REF/RIVET_ANALYSIS_NAME/d03-x01-y01
 Title: Table 3
 Type: Scatter2D
 ---
-# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+
+# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+	
 3.000000e+01	3.000000e+01	3.000000e+01	7.000000e+03	1.102935e+03	1.102935e+03
 8.000000e+01	2.000000e+01	2.000000e+01	9.800000e+03	1.601827e+03	1.601827e+03
 1.500000e+02	5.000000e+01	5.000000e+01	1.600000e+03	4.917164e+02	4.917164e+02
 4.000000e+02	2.000000e+02	2.000000e+02	8.000000e+01	6.003332e+01	6.003332e+01
 END YODA_SCATTER2D_V2
 BEGIN YODA_SCATTER2D_V2 /REF/RIVET_ANALYSIS_NAME/d04-x01-y01
+Variations: [""]
 ErrorBreakdown: {0: {stat: {dn: -3340, up: 3340}, 'sys,background': {dn: -740, up: 740}, 'sys,detector': {dn: -80, up: 80}}, 1: {stat: {dn: -3210, up: 3210}, 'sys,background': {dn: -260, up: 260}, 'sys,detector': {dn: -200, up: 200}}, 2: {stat: {dn: -1490, up: 1490}, 'sys,background': {dn: -390, up: 390}, 'sys,detector': {dn: -120, up: 120}}}
 
 IsRef: 1
 Path: /REF/RIVET_ANALYSIS_NAME/d04-x01-y01
 Title: Table 4
 Type: Scatter2D
 ---
-# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+
+# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+	
 7.000000e+01	2.000000e+01	2.000000e+01	9.930000e+03	3.421929e+03	3.421929e+03
 1.100000e+02	2.000000e+01	2.000000e+01	8.280000e+03	3.226717e+03	3.226717e+03
 1.650000e+02	3.500000e+01	3.500000e+01	3.900000e+03	1.544862e+03	1.544862e+03
 END YODA_SCATTER2D_V2
 BEGIN YODA_SCATTER2D_V2 /REF/RIVET_ANALYSIS_NAME/d05-x01-y01
+Variations: [""]
 ErrorBreakdown: {0: {stat: {dn: -69000, up: 69000}, 'sys,background': {dn: -10, up: 10}, 'sys,detector': {dn: -6600, up: 6600}}, 1: {stat: {dn: -100000, up: 100000}, 'sys,background': {dn: -400, up: 400}, 'sys,detector': {dn: -9000, up: 9000}}, 2: {stat: {dn: -80000, up: 80000}, 'sys,background': {dn: -300, up: 300}, 'sys,detector': {dn: -10000, up: 10000}}, 3: {stat: {dn: -50000, up: 50000}, 'sys,background': {dn: -300, up: 300}, 'sys,detector': {dn: -2000, up: 2000}}}
 
 IsRef: 1
 Path: /REF/RIVET_ANALYSIS_NAME/d05-x01-y01
 Title: Table 5
 Type: Scatter2D
 ---
-# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+
+# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+	
 2.500000e-01	2.500000e-01	2.500000e-01	1.300000e+05	6.931493e+04	6.931493e+04
 7.500000e-01	2.500000e-01	2.500000e-01	2.800000e+05	1.004050e+05	1.004050e+05
 1.350000e+00	3.500000e-01	3.500000e-01	2.600000e+05	8.062314e+04	8.062314e+04
 2.420795e+00	7.207950e-01	7.207950e-01	4.200000e+05	5.004088e+04	5.004088e+04
 END YODA_SCATTER2D_V2
 BEGIN YODA_SCATTER2D_V2 /REF/RIVET_ANALYSIS_NAME/d06-x01-y01
+Variations: [""]
 ErrorBreakdown: {0: {stat: {dn: -4400, up: 4400}, 'sys,background': {dn: -1900, up: 1900}, 'sys,detector': {dn: -300, up: 300}}, 1: {stat: {dn: -2630, up: 2630}, 'sys,background': {dn: -280, up: 280}, 'sys,detector': {dn: -230, up: 230}}, 2: {stat: {dn: -1210, up: 1210}, 'sys,background': {dn: -480, up: 480}, 'sys,detector': {dn: -60, up: 60}}}
 
 IsRef: 1
 Path: /REF/RIVET_ANALYSIS_NAME/d06-x01-y01
 Title: Table 8
 Type: Scatter2D
 ---
-# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+
+# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+	
 2.350000e+02	1.500000e+01	1.500000e+01	1.050000e+04	4.802083e+03	4.802083e+03
 2.750000e+02	2.500000e+01	2.500000e+01	6.320000e+03	2.654845e+03	2.654845e+03
 3.500000e+02	5.000000e+01	5.000000e+01	3.680000e+03	1.303112e+03	1.303112e+03
 END YODA_SCATTER2D_V2
 BEGIN YODA_SCATTER2D_V2 /REF/RIVET_ANALYSIS_NAME/d07-x01-y01
+Variations: [""]
 ErrorBreakdown: {}
 
 IsRef: 1
 Path: /REF/RIVET_ANALYSIS_NAME/d07-x01-y01
 Title: Table 9
 Type: Scatter2D
 ---
-# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+
+# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+	
 2.125000e+02	1.250000e+01	1.250000e+01	0.000000e+00	0.000000e+00	0.000000e+00
 2.375000e+02	1.250000e+01	1.250000e+01	6.000000e+00	0.000000e+00	0.000000e+00
 2.625000e+02	1.250000e+01	1.250000e+01	1.000000e+00	0.000000e+00	0.000000e+00
@@ -128,14 +135,15 @@ Type: Scatter2D
 4.875000e+02	1.250000e+01	1.250000e+01	2.000000e+00	0.000000e+00	0.000000e+00
 END YODA_SCATTER2D_V2
 BEGIN YODA_SCATTER2D_V2 /REF/RIVET_ANALYSIS_NAME/d07-x01-y02
+Variations: [""]
 ErrorBreakdown: {0: {error: {dn: -0.0, up: 0.0}}, 1: {error: {dn: -0.51, up: 0.41}}, 2: {error: {dn: -0.26, up: 0.41}}, 3: {error: {dn: -0.19, up: 0.12}}, 4: {error: {dn: -0.15, up: 0.16}}, 5: {error: {dn: -0.16, up: 0.19}}, 6: {error: {dn: -0.55, up: 0.56}}, 7: {error: {dn: -0.15, up: 0.1}}, 8: {error: {dn: -0.1, up: 0.11}}, 9: {error: {dn: -0.08, up: 0.08}}, 10: {error: {dn: -0.33, up: 0.33}}, 11: {error: {dn: -0.14, up: 0.17}}}
 
 IsRef: 1
 Path: /REF/RIVET_ANALYSIS_NAME/d07-x01-y02
 Title: Table 9
 Type: Scatter2D
 ---
-# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+
+# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+	
 2.125000e+02	1.250000e+01	1.250000e+01	0.000000e+00	0.000000e+00	0.000000e+00
 2.375000e+02	1.250000e+01	1.250000e+01	9.500000e-01	5.100000e-01	4.100000e-01
 2.625000e+02	1.250000e+01	1.250000e+01	9.000000e-01	2.600000e-01	4.100000e-01
@@ -150,14 +158,15 @@ Type: Scatter2D
 4.875000e+02	1.250000e+01	1.250000e+01	1.600000e-01	1.400000e-01	1.700000e-01
 END YODA_SCATTER2D_V2
 BEGIN YODA_SCATTER2D_V2 /REF/RIVET_ANALYSIS_NAME/d07-x01-y03
+Variations: [""]
 ErrorBreakdown: {}
 
 IsRef: 1
 Path: /REF/RIVET_ANALYSIS_NAME/d07-x01-y03
 Title: Table 9
 Type: Scatter2D
 ---
-# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+
+# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+	
 2.125000e+02	1.250000e+01	1.250000e+01	0.000000e+00	0.000000e+00	0.000000e+00
 2.375000e+02	1.250000e+01	1.250000e+01	6.460000e+00	0.000000e+00	0.000000e+00
 2.625000e+02	1.250000e+01	1.250000e+01	6.820000e+00	0.000000e+00	0.000000e+00
@@ -172,14 +181,15 @@ Type: Scatter2D
 4.875000e+02	1.250000e+01	1.250000e+01	9.300000e-01	0.000000e+00	0.000000e+00
 END YODA_SCATTER2D_V2
 BEGIN YODA_SCATTER2D_V2 /REF/RIVET_ANALYSIS_NAME/d07-x01-y04
+Variations: [""]
 ErrorBreakdown: {}
 
 IsRef: 1
 Path: /REF/RIVET_ANALYSIS_NAME/d07-x01-y04
 Title: Table 9
 Type: Scatter2D
 ---
-# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+
+# xval	 xerr-	 xerr+	 yval	 yerr-	 yerr+	
 2.125000e+02	1.250000e+01	1.250000e+01	0.000000e+00	0.000000e+00	0.000000e+00
 2.375000e+02	1.250000e+01	1.250000e+01	9.700000e-01	0.000000e+00	0.000000e+00
 2.625000e+02	1.250000e+01	1.250000e+01	1.070000e+00	0.000000e+00	0.000000e+00

hepdata_converter/writers/array_writer.py

@@ -22,7 +22,11 @@ def is_number_var(cls, *variables):
         for variable in variables:
             for element in variable['values']:
                 if 'value' in element and isinstance(element['value'], (str, unicode)):
-                    is_number_list.append(False)
+                    try:
+                        element['value'] = float(element['value'])
+                        is_number_list.append(True)
+                    except ValueError:
+                        is_number_list.append(False)
                 else:
                     is_number_list.append(True)
         return is_number_list
@@ -171,12 +175,16 @@ def calculate_total_errors(variable, is_number_list, min_errs, max_errs, values,
                         label = error.get('label', 'error')
                         err_breakdown[i][label] = {}
                         if 'asymerror' in error:
-                            err_minus = error_value_processor(entry['value'], error['asymerror']['minus'])
-                            err_plus = error_value_processor(entry['value'], error['asymerror']['plus'])
-                            errors_min += pow(min(err_plus, err_minus, 0.0), 2)
-                            errors_max += pow(max(err_plus, err_minus, 0.0), 2)
-                            err_breakdown[i][label]['up'] = err_plus # want to maintain directionality of errors
-                            err_breakdown[i][label]['dn'] = err_minus # want to maintain directionality of errors
+                            try:
+                                err_minus = error_value_processor(entry['value'], error['asymerror']['minus'])
+                                err_plus = error_value_processor(entry['value'], error['asymerror']['plus'])
+                                errors_min += pow(min(err_plus, err_minus, 0.0), 2)
+                                errors_max += pow(max(err_plus, err_minus, 0.0), 2)
+                                err_breakdown[i][label]['up'] = err_plus # want to maintain directionality of errors
+                                err_breakdown[i][label]['dn'] = err_minus # want to maintain directionality of errors
+                            except TypeError:
+                                log.error('TypeError encountered when parsing {0} and {1}'.format(
+                                    error['asymerror']['minus'], error['asymerror']['plus']))
                         elif 'symerror' in error:
                             try:
                                 err = error_value_processor(entry['value'], error['symerror'])
@@ -185,7 +193,7 @@ def calculate_total_errors(variable, is_number_list, min_errs, max_errs, values,
                                 err_breakdown[i][label]['up'] = err
                                 err_breakdown[i][label]['dn'] = -err
                             except TypeError:
-                                print log.error('TypeError encountered when parsing {0}'.format(error['symerror']))
+                                log.error('TypeError encountered when parsing {0}'.format(error['symerror']))
 
                     min_errs.append(sqrt(errors_min))
                     max_errs.append(sqrt(errors_max))

hepdata_converter/writers/root_writer.py

@@ -32,8 +32,9 @@ def match(cls, independent_variables_map, dependent_variable):
             for independent_variable in independent_variables_map:
                 if not independent_variable['values']:
                     return False
-                if 'low' not in independent_variable['values'][0] or 'high' not in independent_variable['values'][0]:
-                    return False
+                for value in independent_variable['values']:
+                    if 'low' not in value or 'high' not in value:
+                        return False
             return True
         return False
 
@@ -44,23 +45,18 @@ def _create_empty_hist(self, dependent_var_title, index, yval):
             xval.append([])
             i_var = self.independent_variables[i]['values']
             for x in i_var:
-                xval[i].append(x['low'])
-            xval[i].append(i_var[-1]['high'])
+                if x['low'] not in xval[i]:
+                    xval[i].append(x['low'])
+                if x['high'] not in xval[i]:
+                    xval[i].append(x['high'])
 
         name = "Hist%sD_y%s_e%s" % (self.dim, self.dependent_variable_index + 1, index)
 
         # order bin values of independent variables
         xval_ordered = []
         for i in xrange(self.dim):
             xval_ordered.append([])
-            for j, x in enumerate(xval[i]):
-                if j == 0:
-                    x_highest = x
-                    xval_ordered[i].append(x)
-                else:
-                    if x > x_highest:
-                        x_highest = x
-                        xval_ordered[i].append(x)
+            xval_ordered[i] = sorted(xval[i])
 
         if 1 == self.dim:
             nbinsx = len(xval_ordered[0]) - 1
@@ -110,14 +106,7 @@ def _create_hist(self, xval):
         xval_ordered = []
         for i in xrange(self.dim):
             xval_ordered.append([])
-            for j, x in enumerate(xval[i]):
-                if j == 0:
-                    x_highest = x
-                    xval_ordered[i].append(x)
-                else:
-                    if x > x_highest:
-                        x_highest = x
-                        xval_ordered[i].append(x)
+            xval_ordered[i] = sorted(xval[i])
 
         if 1 == self.dim:
             nbinsx = len(xval_ordered[0]) - 1
@@ -247,8 +236,10 @@ def create_objects(self):
             xval.append([])
             i_var = self.independent_variables[i]['values']
             for x in i_var:
-                xval[i].append(x['low'])
-            xval[i].append(i_var[-1]['high'])
+                if x['low'] not in xval[i]:
+                    xval[i].append(x['low'])
+                if x['high'] not in xval[i]:
+                    xval[i].append(x['high'])
 
         try:
             hist = self._create_hist(xval)

hepdata_converter/writers/utils.py

@@ -9,11 +9,14 @@ def error_value_processor(value, error):
     :return: the absolute error, e.g. 12 for the above case.
     """
     if type(error) is str:
-        if "%" in error:
-            error = error.replace("%", "")
-            error = float(error)
-
-            error_abs = (value/100) * error
-            return error_abs
+        try:
+            if "%" in error:
+                error_float = float(error.replace("%", ""))
+                error_abs = (value/100) * error_float
+                return error_abs
+            else:
+                error = float(error)
+        except:
+            pass
 
     return error