Use Pytest for test execution

.appveyor.yml

@@ -19,4 +19,4 @@ install:
   - python -m pip install --user ".[dev]"
 
 test_script:
-  - python -m nose --tests benchexec.tablegenerator
+  - python -m pytest benchexec/tablegenerator/

.gitlab-ci.yml

@@ -50,7 +50,7 @@ stages:
   script:
     - sudo -u $PRIMARY_USER
         COVERAGE_PROCESS_START=.coveragerc
-        coverage run -m nose
+        coverage run -m pytest
   after_script:
     - sudo -u $PRIMARY_USER coverage combine
     - sudo -u $PRIMARY_USER coverage report

benchexec/tablegenerator/statistics.py

@@ -14,6 +14,9 @@
 from benchexec.tablegenerator import util
 from benchexec.tablegenerator.columns import ColumnType
 
+# It's important to make sure on *all* entry points / methods which perform arithmetics that the correct
+# rounding / context is used.
+DECIMAL_CONTEXT = decimal.Context(rounding=decimal.ROUND_HALF_UP)
 
 nan = Decimal("nan")
 inf = Decimal("inf")
@@ -65,55 +68,56 @@ def __str__(self):
 
     @classmethod
     def from_list(cls, values):
-        if any(v is not None and v.is_nan() for v in values):
-            return StatValue(nan, nan, nan, nan, nan, nan)
-
-        values = sorted(v for v in values if v is not None)
-        if not values:
-            return None
-
-        values_len = len(values)
-        min_value = values[0]
-        max_value = values[-1]
-
-        if min_value == -inf and max_value == +inf:
-            values_sum = nan
-            mean = nan
-            stdev = nan
-        elif max_value == inf:
-            values_sum = inf
-            mean = inf
-            stdev = inf
-        elif min_value == -inf:
-            values_sum = -inf
-            mean = -inf
-            stdev = inf
-        else:
-            values_sum = sum(values)
-            mean = values_sum / values_len
-
-            # The scaling is just to avoid having too few decimal digits when printing,
-            # the value is still just 0.
-            stdev = Decimal(0).scaleb(-decimal.getcontext().prec)
-            for v in values:
-                diff = v - mean
-                stdev += diff * diff
-            stdev = (stdev / values_len).sqrt()
-
-        half, len_is_odd = divmod(values_len, 2)
-        if len_is_odd:
-            median = values[half]
-        else:
-            median = (values[half - 1] + values[half]) / Decimal(2)
-
-        return StatValue(
-            values_sum,
-            min=min_value,
-            max=max_value,
-            avg=mean,
-            median=median,
-            stdev=stdev,
-        )
+        with decimal.localcontext(DECIMAL_CONTEXT):
+            if any(v is not None and v.is_nan() for v in values):
+                return StatValue(nan, nan, nan, nan, nan, nan)
+
+            values = sorted(v for v in values if v is not None)
+            if not values:
+                return None
+
+            values_len = len(values)
+            min_value = values[0]
+            max_value = values[-1]
+
+            if min_value == -inf and max_value == +inf:
+                values_sum = nan
+                mean = nan
+                stdev = nan
+            elif max_value == inf:
+                values_sum = inf
+                mean = inf
+                stdev = inf
+            elif min_value == -inf:
+                values_sum = -inf
+                mean = -inf
+                stdev = inf
+            else:
+                values_sum = sum(values)
+                mean = values_sum / values_len
+
+                # The scaling is just to avoid having too few decimal digits when printing,
+                # the value is still just 0.
+                stdev = Decimal(0).scaleb(-decimal.getcontext().prec)
+                for v in values:
+                    diff = v - mean
+                    stdev += diff * diff
+                stdev = (stdev / values_len).sqrt()
+
+            half, len_is_odd = divmod(values_len, 2)
+            if len_is_odd:
+                median = values[half]
+            else:
+                median = (values[half - 1] + values[half]) / Decimal(2)
+
+            return StatValue(
+                values_sum,
+                min=min_value,
+                max=max_value,
+                avg=mean,
+                median=median,
+                stdev=stdev,
+            )
 
 
 def get_stats_of_run_set(runResults, correct_only):

benchexec/tablegenerator/test_columns.py

@@ -182,6 +182,15 @@ def test_format_value_small_value(self):
         formatted_value_aligned = small_value_column.format_value("2", "html_cell")
         self.assertEqual(formatted_value_aligned, ".0000000002  ")
 
+    def test_invalid_rounding_mode(self):
+        import decimal
+
+        decimal.getcontext().rounding = decimal.ROUND_HALF_DOWN
+        formatted_value_no_align_zeros_cut = self.measure_column.format_value(
+            "5.7715", *self.default_optionals
+        )
+        self.assertEqual(formatted_value_no_align_zeros_cut, "5.772")
+
     def test_format_value_align_int(self):
         formatted_value_int_aligned = self.measure_column.format_value(
             "20", "html_cell"

benchexec/tablegenerator/test_integration/__init__.py

@@ -17,8 +17,6 @@
 import benchexec.util
 import benchexec.tablegenerator.util
 
-sys.dont_write_bytecode = True  # prevent creation of .pyc files
-
 here = os.path.relpath(os.path.dirname(__file__))
 base_dir = os.path.join(here, "..", "..", "..")
 bin_dir = os.path.join(base_dir, "bin")
@@ -38,11 +36,6 @@ class TableGeneratorIntegrationTests(unittest.TestCase):
     # Tests compare the generated CSV files and ignore the HTML files
     # because we assume the HTML files change more often on purpose.
 
-    @classmethod
-    def setUpClass(cls):
-        cls.longMessage = True
-        cls.maxDiff = None
-
     def setUp(self):
         # We use a temporary directory inside the source tree to avoid mismatching
         # path names inside HTML tables.

benchexec/tablegenerator/test_statvalue.py

@@ -6,20 +6,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from decimal import Decimal
-import sys
 import unittest
 
 from benchexec.tablegenerator.statistics import StatValue
 
-sys.dont_write_bytecode = True  # prevent creation of .pyc files
-
 
 class TestStatValue(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.longMessage = True
-        cls.maxDiff = None
-
     def test_empty(self):
         self.assertIsNone(StatValue.from_list([]))
 

benchexec/tablegenerator/test_util.py

@@ -6,19 +6,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from decimal import Decimal
-import sys
 import unittest
 
 from benchexec.tablegenerator import util
 
-sys.dont_write_bytecode = True  # prevent creation of .pyc files
-
 
 class TestUnit(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.longMessage = True
-        cls.maxDiff = None
 
     def assertEqualNumberAndUnit(self, value, number, unit):
         self.assertEqual(util.split_number_and_unit(value), (number, unit))

benchexec/test_analyze_run_result.py

@@ -5,8 +5,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-import logging
-import sys
 import unittest
 import types
 
@@ -20,17 +18,10 @@
 )
 from benchexec.tools.template import BaseTool
 
-sys.dont_write_bytecode = True  # prevent creation of .pyc files
-
 normal_result = ProcessExitCode(raw=0, value=0, signal=None)
 
 
 class TestResult(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.longMessage = True
-        logging.disable(logging.CRITICAL)
-
     def create_run(self, info_result=RESULT_UNKNOWN):
         runSet = types.SimpleNamespace()
         runSet.log_folder = "."

benchexec/test_benchmark_definition.py

@@ -69,10 +69,6 @@ class TestBenchmarkDefinition(unittest.TestCase):
     testing mostly the classes from benchexec.model.
     """
 
-    @classmethod
-    def setUpClass(cls):
-        cls.longMessage = True
-
     @patch("benchexec.model.load_task_definition_file", new=mock_load_task_def_file)
     @patch("benchexec.result.Property.create", new=mock_property_create)
     @patch("benchexec.util.expand_filename_pattern", new=mock_expand_filename_pattern)

benchexec/test_cgroups.py

@@ -5,23 +5,13 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-import logging
 import subprocess
-import sys
 import unittest
 
 from benchexec import check_cgroups
 
-sys.dont_write_bytecode = True  # prevent creation of .pyc files
-
 
 class TestCheckCgroups(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.longMessage = True
-        cls.maxDiff = None
-        logging.disable(logging.CRITICAL)
-
     def execute_run_extern(self, *args, **kwargs):
         try:
             return subprocess.check_output(

benchexec/test_core_assignment.py

@@ -6,25 +6,17 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import itertools
-import logging
-import sys
 import unittest
 import math
 
 from benchexec.resources import _get_cpu_cores_per_run0
 
-sys.dont_write_bytecode = True  # prevent creation of .pyc files
-
 
 def lrange(start, end):
     return list(range(start, end))
 
 
 class TestCpuCoresPerRun(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.longMessage = True
-        logging.disable(logging.CRITICAL)
 
     def assertValid(self, coreLimit, num_of_threads, expectedResult=None):
         result = _get_cpu_cores_per_run0(

benchexec/test_integration/__init__.py

@@ -10,15 +10,12 @@
 import os
 import shutil
 import subprocess
-import sys
 import tempfile
 import unittest
 import zipfile
 
 from xml.etree import ElementTree
 
-sys.dont_write_bytecode = True  # prevent creation of .pyc files
-
 here = os.path.dirname(__file__)
 base_dir = os.path.join(here, "..", "..")
 bin_dir = os.path.join(base_dir, "bin")
@@ -45,10 +42,6 @@
 
 
 class BenchExecIntegrationTests(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.longMessage = True
-        cls.maxDiff = None
 
     def _build_tmp_dir(self):
         """

benchexec/test_pqos.py

@@ -10,13 +10,11 @@
 """
 import json
 import copy
-import logging
 import unittest
 from subprocess import CalledProcessError
 from unittest.mock import patch, MagicMock
 from benchexec.pqos import Pqos
 
-
 mock_pqos_wrapper_output = {
     "load_pqos": {
         "function_output": {},
@@ -147,10 +145,6 @@ class TestPqos(unittest.TestCase):
     Unit tests for pqos module
     """
 
-    @classmethod
-    def setUpClass(cls):
-        logging.disable(logging.CRITICAL)
-
     @patch("benchexec.pqos.find_executable2", return_value="/path/to/pqos_wrapper/lib")
     def test_pqos_init(self, mock_find_executable):
         """

benchexec/test_result.py

@@ -5,8 +5,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-import logging
-import sys
 import tempfile
 import unittest
 
@@ -18,8 +16,6 @@
     _SCORE_WRONG_FALSE,
 )
 
-sys.dont_write_bytecode = True  # prevent creation of .pyc files
-
 
 class TestExpectedResult(unittest.TestCase):
     def test_via_string(self):
@@ -56,10 +52,6 @@ def test(s):
 
 
 class TestResult(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.longMessage = True
-        logging.disable(logging.CRITICAL)
 
     def expected_result(self, result, subcategory=None):
         return {"dummy.prp": ExpectedResult(result, subcategory)}