Merge branch 'rl4sem' into develop

Conflicts:
	CHANGELOG.md

CHANGELOG.md

@@ -16,11 +16,22 @@
   * `ColumnGenerator`: add method `to_feature_generator`
 * `evaluation`:
   * `MultiDataEvaluation`: Add option to supply test data (without using splitting)
+  * `VectorRegressionModelEvaluator`: Handle output column mismatch between model output and ground truth 
+    for the case where there is only a single column, avoiding the exception and issuing a
+    warning instead
 * `dft`:
   * `DFTNormalisation.RuleTemplate`: Add attributes `fit` and `array_valued`
 * `util.deprecation`: Apply `functools.wrap` to retain meta-data of wrapped function
 * `util.logging`: 
   * Support multiple configuration callbacks in `set_configure_callback` 
+  * Add line number to default format (`LOG_DEFAULT_FORMAT`)
+  * Add function `is_enabled` to check whether a log handler is registered
+  * Add context manager `LoggingDisabledContext` to temporarily disable logging
+* `util.io`:
+  * `ResultWriter`:
+    * Allow to disable an instance such that no results are written (constructor parameter `enabled`)
+    * Add default configuration for closing figures after writing them (constructor parameter `close_figures`)
+    * `write_image`: Improve layout in written images by setting `bbox_inches='tight'`
 
 ### Fixes
 

src/sensai/evaluation/evaluator.py

@@ -317,7 +317,16 @@ def _eval_model(self, model: VectorRegressionModel, data: InputOutputData) -> Ve
         eval_stats_by_var_name = {}
         predictions, ground_truth = self._compute_outputs(model, data)
         for predictedVarName in predictions.columns:
-            eval_stats = RegressionEvalStats(y_predicted=predictions[predictedVarName], y_true=ground_truth[predictedVarName],
+            if predictedVarName in ground_truth.columns:
+                y_true = ground_truth[predictedVarName]
+            else:
+                if len(predictions.columns) == 1 and len(ground_truth.columns) == 1:
+                    log.warning(f"Model output column '{predictedVarName}' does not match ground truth column '{ground_truth.columns[0]}'; "
+                        f"assuming that this is not a problem since there is but a single column available")
+                    y_true = ground_truth.iloc[:, 0]
+                else:
+                    raise Exception(f"Model output column '{predictedVarName}' not found in ground truth columns {ground_truth.columns}")
+            eval_stats = RegressionEvalStats(y_predicted=predictions[predictedVarName], y_true=y_true,
                 metrics=self.params.metrics,
                 additional_metrics=self.params.additional_metrics,
                 model=model,

src/sensai/util/io.py

@@ -1,7 +1,7 @@
 import io
 import logging
 import os
-from typing import Sequence, Optional, Tuple, List
+from typing import Sequence, Optional, Tuple, List, Any
 
 import matplotlib.figure
 from matplotlib import pyplot as plt
@@ -13,10 +13,19 @@
 class ResultWriter:
     log = log.getChild(__qualname__)
 
-    def __init__(self, result_dir, filename_prefix=""):
+    def __init__(self, result_dir, filename_prefix="", enabled: bool = True, close_figures: bool = False):
+        """
+        :param result_dir:
+        :param filename_prefix:
+        :param enabled: whether the result writer is enabled; if it is not, it will create neither files nor directories
+        :param close_figures: whether to close figures that are passed by default
+        """
         self.result_dir = result_dir
-        os.makedirs(result_dir, exist_ok=True)
         self.filename_prefix = filename_prefix
+        self.enabled = enabled
+        self.close_figures_default = close_figures
+        if self.enabled:
+            os.makedirs(result_dir, exist_ok=True)
 
     def child_with_added_prefix(self, prefix: str) -> "ResultWriter":
         """
@@ -26,14 +35,15 @@ def child_with_added_prefix(self, prefix: str) -> "ResultWriter":
         :param prefix: the prefix to append
         :return: a new writer instance
         """
-        return ResultWriter(self.result_dir, filename_prefix=self.filename_prefix + prefix)
+        return ResultWriter(self.result_dir, filename_prefix=self.filename_prefix + prefix, enabled=self.enabled,
+            close_figures=self.close_figures_default)
 
-    def child_for_subdirectory(self, dir_name: str):
+    def child_for_subdirectory(self, dir_name: str) -> "ResultWriter":
         result_dir = os.path.join(self.result_dir, dir_name)
-        os.makedirs(result_dir, exist_ok=True)
-        return ResultWriter(result_dir, filename_prefix=self.filename_prefix)
+        return ResultWriter(result_dir, filename_prefix=self.filename_prefix, enabled=self.enabled,
+            close_figures=self.close_figures_default)
 
-    def path(self, filename_suffix: str, extension_to_add=None, valid_other_extensions: Optional[Sequence[str]] = None):
+    def path(self, filename_suffix: str, extension_to_add=None, valid_other_extensions: Optional[Sequence[str]] = None) -> str:
         """
         :param filename_suffix: the suffix to add (which may or may not already include a file extension)
         :param extension_to_add: if not None, the file extension to add (without the leading ".") unless
@@ -56,55 +66,62 @@ def path(self, filename_suffix: str, extension_to_add=None, valid_other_extensio
         path = os.path.join(self.result_dir, f"{self.filename_prefix}{filename_suffix}")
         return path
 
-    def write_text_file(self, filename_suffix, content):
+    def write_text_file(self, filename_suffix: str, content: str):
         p = self.path(filename_suffix, extension_to_add="txt")
-        self.log.info(f"Saving text file {p}")
-        with open(p, "w") as f:
-            f.write(content)
+        if self.enabled:
+            self.log.info(f"Saving text file {p}")
+            with open(p, "w") as f:
+                f.write(content)
         return p
 
-    def write_text_file_lines(self, filename_suffix, lines: List[str]):
+    def write_text_file_lines(self, filename_suffix: str, lines: List[str]):
         p = self.path(filename_suffix, extension_to_add="txt")
-        self.log.info(f"Saving text file {p}")
-        write_text_file_lines(lines, p)
+        if self.enabled:
+            self.log.info(f"Saving text file {p}")
+            write_text_file_lines(lines, p)
         return p
 
-    def write_data_frame_text_file(self, filename_suffix, df: pd.DataFrame):
+    def write_data_frame_text_file(self, filename_suffix: str, df: pd.DataFrame):
         p = self.path(filename_suffix, extension_to_add="df.txt", valid_other_extensions="txt")
-        self.log.info(f"Saving data frame text file {p}")
-        with open(p, "w") as f:
-            f.write(df.to_string())
+        if self.enabled:
+            self.log.info(f"Saving data frame text file {p}")
+            with open(p, "w") as f:
+                f.write(df.to_string())
         return p
 
-    def write_data_frame_csv_file(self, filename_suffix, df: pd.DataFrame, index=True, header=True):
+    def write_data_frame_csv_file(self, filename_suffix: str, df: pd.DataFrame, index=True, header=True):
         p = self.path(filename_suffix, extension_to_add="csv")
-        self.log.info(f"Saving data frame CSV file {p}")
-        df.to_csv(p, index=index, header=header)
+        if self.enabled:
+            self.log.info(f"Saving data frame CSV file {p}")
+            df.to_csv(p, index=index, header=header)
         return p
 
-    def write_figure(self, filename_suffix, fig, close_figure=False):
+    def write_figure(self, filename_suffix: str, fig: plt.Figure, close_figure: Optional[bool] = None):
         """
         :param filename_suffix: the filename suffix, which may or may not include a file extension, valid extensions being {"png", "jpg"}
         :param fig: the figure to save
-        :param close_figure: whether to close the figure after having saved it
-        :return: the path to the file that was written
+        :param close_figure: whether to close the figure after having saved it; if None, use default passed at construction
+        :return: the path to the file that was written (or would have been written if the writer was enabled)
         """
         p = self.path(filename_suffix, extension_to_add="png", valid_other_extensions=("jpg",))
-        self.log.info(f"Saving figure {p}")
-        fig.savefig(p)
-        if close_figure:
-            plt.close(fig)
+        if self.enabled:
+            self.log.info(f"Saving figure {p}")
+            fig.savefig(p, bbox_inches="tight")
+            must_close_figure = close_figure if close_figure is not None else self.close_figures_default
+            if must_close_figure:
+                plt.close(fig)
         return p
 
     def write_figures(self, figures: Sequence[Tuple[str, matplotlib.figure.Figure]], close_figures=False):
         for name, fig in figures:
             self.write_figure(name, fig, close_figure=close_figures)
 
-    def write_pickle(self, filename_suffix, obj):
+    def write_pickle(self, filename_suffix: str, obj: Any):
         from .pickle import dump_pickle
         p = self.path(filename_suffix, extension_to_add="pickle")
-        self.log.info(f"Saving pickle {p}")
-        dump_pickle(obj, p)
+        if self.enabled:
+            self.log.info(f"Saving pickle {p}")
+            dump_pickle(obj, p)
         return p
 
 

src/sensai/util/logging.py

@@ -11,7 +11,7 @@
 
 log = getLogger(__name__)
 
-LOG_DEFAULT_FORMAT = '%(levelname)-5s %(asctime)-15s %(name)s:%(funcName)s - %(message)s'
+LOG_DEFAULT_FORMAT = '%(levelname)-5s %(asctime)-15s %(name)s:%(funcName)s:%(lineno)d - %(message)s'
 
 # Holds the log format that is configured by the user (using function `configure`), such
 # that it can be reused in other places
@@ -38,6 +38,13 @@ def is_log_handler_active(handler):
     return handler in getLogger().handlers
 
 
+def is_enabled() -> bool:
+    """
+    :return: True if logging is enabled (at least one log handler exists)
+    """
+    return getLogger().hasHandlers()
+
+
 def set_configure_callback(callback: Callable[[], None], append: bool = True) -> None:
     """
     Configures a function to be called when logging is configured, e.g. through :func:`configure, :func:`run_main` or
@@ -96,6 +103,7 @@ def run_main(main_fn: Callable[[], Any], format=LOG_DEFAULT_FORMAT, level=lg.DEB
         log.error("Exception during script execution", exc_info=e)
 
 
+# noinspection PyShadowingBuiltins
 def run_cli(main_fn: Callable[[], Any], format=LOG_DEFAULT_FORMAT, level=lg.DEBUG):
     """
     Configures logging with the given parameters and runs the given main function as a
@@ -334,3 +342,22 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_value, traceback):
         if self._log_handler is not None:
             remove_log_handler(self._log_handler)
+
+
+class LoggingDisabledContext:
+    """
+    A context manager that will temporarily disable logging
+    """
+    def __init__(self, highest_level=CRITICAL):
+        """
+        :param highest_level: the highest level to disable
+        """
+        self._highest_level = highest_level
+        self._previous_level = None
+
+    def __enter__(self):
+        self._previous_level = lg.root.manager.disable
+        lg.disable(self._highest_level)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        lg.disable(self._previous_level)