From fddb5a7186a8c1b0fc9af23377ade2a24939438d Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 4 Apr 2024 15:28:50 +0200 Subject: [PATCH 1/2] Sync rl4sem commit 0526030670d3fe72201356206614803eb0e16735 Author: Dominik Jain Date: Wed Apr 3 13:52:53 2024 +0200 Add line number to LOG_DEFAULT_FORMAT src/sensai/util/logging.py commit 62c1abfcb98a2aca350620d0cabdecfcb0d205d5 Author: Dominik Jain Date: Tue Apr 2 12:52:20 2024 +0200 Add logging.is_enabled src/sensai/util/logging.py commit 8a908ee7cb7f1ddfbd0068e7e5749fdaa3830635 Author: Dominik Jain Date: Wed Mar 27 17:15:13 2024 +0100 Add LoggingDisabledContext src/sensai/util/logging.py commit 662e6046d92630dd29beeb50c6c655dbbaab84ae Author: Dominik Jain Date: Tue Mar 19 14:34:52 2024 +0100 ResultWriter: Pass on close_figures default to children src/sensai/util/io.py commit 88f7196312c9ba1314fd55bef782ce4b81b67bee Author: Dominik Jain Date: Tue Mar 19 13:51:00 2024 +0100 ResultWriter: Add constructor parameter to configure default for figure closing behaviour src/sensai/util/io.py commit 1199fc25c2fab4900b95dfe76a999998186eb15c Author: Dominik Jain Date: Mon Mar 4 16:17:43 2024 +0100 ResultWriter.write_image: Add bbox_inches='tight' src/sensai/util/io.py commit 67ab1cbe290cb6fb6ec176c20d799c5d472792af Author: Dominik Jain Date: Tue Feb 27 16:00:23 2024 +0100 ResultWriter: Add option to disable the instance such that no results are written src/sensai/util/io.py commit 90157f9b30ad46ab5dd9d10c427a1502a20646c4 Author: Dominik Jain Date: Tue Feb 27 15:47:45 2024 +0100 VectorRegressionModelEvaluator: Handle output column mismatch between model output and ground truth for the case where there is only a single column, avoiding the exception and issuing a warning instead src/sensai/evaluation/evaluator.py --- src/sensai/evaluation/evaluator.py | 11 +++- src/sensai/util/io.py | 81 ++++++++++++++++++------------ src/sensai/util/logging.py | 29 ++++++++++- 3 files changed, 87 insertions(+), 34 deletions(-) diff --git a/src/sensai/evaluation/evaluator.py b/src/sensai/evaluation/evaluator.py index 680e46db..3989aabd 100644 --- a/src/sensai/evaluation/evaluator.py +++ b/src/sensai/evaluation/evaluator.py @@ -317,7 +317,16 @@ def _eval_model(self, model: VectorRegressionModel, data: InputOutputData) -> Ve eval_stats_by_var_name = {} predictions, ground_truth = self._compute_outputs(model, data) for predictedVarName in predictions.columns: - eval_stats = RegressionEvalStats(y_predicted=predictions[predictedVarName], y_true=ground_truth[predictedVarName], + if predictedVarName in ground_truth.columns: + y_true = ground_truth[predictedVarName] + else: + if len(predictions.columns) == 1 and len(ground_truth.columns) == 1: + log.warning(f"Model output column '{predictedVarName}' does not match ground truth column '{ground_truth.columns[0]}'; " + f"assuming that this is not a problem since there is but a single column available") + y_true = ground_truth.iloc[:, 0] + else: + raise Exception(f"Model output column '{predictedVarName}' not found in ground truth columns {ground_truth.columns}") + eval_stats = RegressionEvalStats(y_predicted=predictions[predictedVarName], y_true=y_true, metrics=self.params.metrics, additional_metrics=self.params.additional_metrics, model=model, diff --git a/src/sensai/util/io.py b/src/sensai/util/io.py index 94645715..30c99eae 100644 --- a/src/sensai/util/io.py +++ b/src/sensai/util/io.py @@ -1,7 +1,7 @@ import io import logging import os -from typing import Sequence, Optional, Tuple, List +from typing import Sequence, Optional, Tuple, List, Any import matplotlib.figure from matplotlib import pyplot as plt @@ -13,10 +13,19 @@ class ResultWriter: log = log.getChild(__qualname__) - def __init__(self, result_dir, filename_prefix=""): + def __init__(self, result_dir, filename_prefix="", enabled: bool = True, close_figures: bool = False): + """ + :param result_dir: + :param filename_prefix: + :param enabled: whether the result writer is enabled; if it is not, it will create neither files nor directories + :param close_figures: whether to close figures that are passed by default + """ self.result_dir = result_dir - os.makedirs(result_dir, exist_ok=True) self.filename_prefix = filename_prefix + self.enabled = enabled + self.close_figures_default = close_figures + if self.enabled: + os.makedirs(result_dir, exist_ok=True) def child_with_added_prefix(self, prefix: str) -> "ResultWriter": """ @@ -26,14 +35,15 @@ def child_with_added_prefix(self, prefix: str) -> "ResultWriter": :param prefix: the prefix to append :return: a new writer instance """ - return ResultWriter(self.result_dir, filename_prefix=self.filename_prefix + prefix) + return ResultWriter(self.result_dir, filename_prefix=self.filename_prefix + prefix, enabled=self.enabled, + close_figures=self.close_figures_default) - def child_for_subdirectory(self, dir_name: str): + def child_for_subdirectory(self, dir_name: str) -> "ResultWriter": result_dir = os.path.join(self.result_dir, dir_name) - os.makedirs(result_dir, exist_ok=True) - return ResultWriter(result_dir, filename_prefix=self.filename_prefix) + return ResultWriter(result_dir, filename_prefix=self.filename_prefix, enabled=self.enabled, + close_figures=self.close_figures_default) - def path(self, filename_suffix: str, extension_to_add=None, valid_other_extensions: Optional[Sequence[str]] = None): + def path(self, filename_suffix: str, extension_to_add=None, valid_other_extensions: Optional[Sequence[str]] = None) -> str: """ :param filename_suffix: the suffix to add (which may or may not already include a file extension) :param extension_to_add: if not None, the file extension to add (without the leading ".") unless @@ -56,55 +66,62 @@ def path(self, filename_suffix: str, extension_to_add=None, valid_other_extensio path = os.path.join(self.result_dir, f"{self.filename_prefix}{filename_suffix}") return path - def write_text_file(self, filename_suffix, content): + def write_text_file(self, filename_suffix: str, content: str): p = self.path(filename_suffix, extension_to_add="txt") - self.log.info(f"Saving text file {p}") - with open(p, "w") as f: - f.write(content) + if self.enabled: + self.log.info(f"Saving text file {p}") + with open(p, "w") as f: + f.write(content) return p - def write_text_file_lines(self, filename_suffix, lines: List[str]): + def write_text_file_lines(self, filename_suffix: str, lines: List[str]): p = self.path(filename_suffix, extension_to_add="txt") - self.log.info(f"Saving text file {p}") - write_text_file_lines(lines, p) + if self.enabled: + self.log.info(f"Saving text file {p}") + write_text_file_lines(lines, p) return p - def write_data_frame_text_file(self, filename_suffix, df: pd.DataFrame): + def write_data_frame_text_file(self, filename_suffix: str, df: pd.DataFrame): p = self.path(filename_suffix, extension_to_add="df.txt", valid_other_extensions="txt") - self.log.info(f"Saving data frame text file {p}") - with open(p, "w") as f: - f.write(df.to_string()) + if self.enabled: + self.log.info(f"Saving data frame text file {p}") + with open(p, "w") as f: + f.write(df.to_string()) return p - def write_data_frame_csv_file(self, filename_suffix, df: pd.DataFrame, index=True, header=True): + def write_data_frame_csv_file(self, filename_suffix: str, df: pd.DataFrame, index=True, header=True): p = self.path(filename_suffix, extension_to_add="csv") - self.log.info(f"Saving data frame CSV file {p}") - df.to_csv(p, index=index, header=header) + if self.enabled: + self.log.info(f"Saving data frame CSV file {p}") + df.to_csv(p, index=index, header=header) return p - def write_figure(self, filename_suffix, fig, close_figure=False): + def write_figure(self, filename_suffix: str, fig: plt.Figure, close_figure: Optional[bool] = None): """ :param filename_suffix: the filename suffix, which may or may not include a file extension, valid extensions being {"png", "jpg"} :param fig: the figure to save - :param close_figure: whether to close the figure after having saved it - :return: the path to the file that was written + :param close_figure: whether to close the figure after having saved it; if None, use default passed at construction + :return: the path to the file that was written (or would have been written if the writer was enabled) """ p = self.path(filename_suffix, extension_to_add="png", valid_other_extensions=("jpg",)) - self.log.info(f"Saving figure {p}") - fig.savefig(p) - if close_figure: - plt.close(fig) + if self.enabled: + self.log.info(f"Saving figure {p}") + fig.savefig(p, bbox_inches="tight") + must_close_figure = close_figure if close_figure is not None else self.close_figures_default + if must_close_figure: + plt.close(fig) return p def write_figures(self, figures: Sequence[Tuple[str, matplotlib.figure.Figure]], close_figures=False): for name, fig in figures: self.write_figure(name, fig, close_figure=close_figures) - def write_pickle(self, filename_suffix, obj): + def write_pickle(self, filename_suffix: str, obj: Any): from .pickle import dump_pickle p = self.path(filename_suffix, extension_to_add="pickle") - self.log.info(f"Saving pickle {p}") - dump_pickle(obj, p) + if self.enabled: + self.log.info(f"Saving pickle {p}") + dump_pickle(obj, p) return p diff --git a/src/sensai/util/logging.py b/src/sensai/util/logging.py index 39286aaf..901ac4db 100644 --- a/src/sensai/util/logging.py +++ b/src/sensai/util/logging.py @@ -11,7 +11,7 @@ log = getLogger(__name__) -LOG_DEFAULT_FORMAT = '%(levelname)-5s %(asctime)-15s %(name)s:%(funcName)s - %(message)s' +LOG_DEFAULT_FORMAT = '%(levelname)-5s %(asctime)-15s %(name)s:%(funcName)s:%(lineno)d - %(message)s' # Holds the log format that is configured by the user (using function `configure`), such # that it can be reused in other places @@ -38,6 +38,13 @@ def is_log_handler_active(handler): return handler in getLogger().handlers +def is_enabled() -> bool: + """ + :return: True if logging is enabled (at least one log handler exists) + """ + return getLogger().hasHandlers() + + def set_configure_callback(callback: Callable[[], None], append: bool = True) -> None: """ Configures a function to be called when logging is configured, e.g. through :func:`configure, :func:`run_main` or @@ -96,6 +103,7 @@ def run_main(main_fn: Callable[[], Any], format=LOG_DEFAULT_FORMAT, level=lg.DEB log.error("Exception during script execution", exc_info=e) +# noinspection PyShadowingBuiltins def run_cli(main_fn: Callable[[], Any], format=LOG_DEFAULT_FORMAT, level=lg.DEBUG): """ Configures logging with the given parameters and runs the given main function as a @@ -334,3 +342,22 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): if self._log_handler is not None: remove_log_handler(self._log_handler) + + +class LoggingDisabledContext: + """ + A context manager that will temporarily disable logging + """ + def __init__(self, highest_level=CRITICAL): + """ + :param highest_level: the highest level to disable + """ + self._highest_level = highest_level + self._previous_level = None + + def __enter__(self): + self._previous_level = lg.root.manager.disable + lg.disable(self._highest_level) + + def __exit__(self, exc_type, exc_value, traceback): + lg.disable(self._previous_level) From bc9e8290bd5bfc32a6ae268740089328babcf4d5 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 4 Apr 2024 15:36:34 +0200 Subject: [PATCH 2/2] Update change log --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c7a2fda..5b5fa4e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,22 @@ * `ColumnGenerator`: add method `to_feature_generator` * `evaluation`: * `MultiDataEvaluationUtil`: Add option to supply test data (without using splitting) + * `VectorRegressionModelEvaluator`: Handle output column mismatch between model output and ground truth + for the case where there is only a single column, avoiding the exception and issuing a + warning instead * `dft`: * `DFTNormalisation.RuleTemplate`: Add attributes `fit` and `array_valued` * `util.deprecation`: Apply `functools.wrap` to retain meta-data of wrapped function * `util.logging`: * Support multiple configuration callbacks in `set_configure_callback` + * Add line number to default format (`LOG_DEFAULT_FORMAT`) + * Add function `is_enabled` to check whether a log handler is registered + * Add context manager `LoggingDisabledContext` to temporarily disable logging +* `util.io`: + * `ResultWriter`: + * Allow to disable an instance such that no results are written (constructor parameter `enabled`) + * Add default configuration for closing figures after writing them (constructor parameter `close_figures`) + * `write_image`: Improve layout in written images by setting `bbox_inches='tight'` ### Fixes