Log module API (aimclub#93)

* serialization fix * bug fix * mixtures were excluded from list of serizalization & bug with extensions fixed * log mod API * making selbst.ini removed * log API & docs * net plot for jupiter * Docs update Fromal style Co-authored-by: Jerzy Kamiński <[email protected]> * Code imprvm Getting rid of nested loops Co-authored-by: Jerzy Kamiński <[email protected]> * log update --------- Co-authored-by: Jerzy Kamiński <[email protected]>
jrzkaminski · Dec 27, 2023 · 77ffcf6 · 77ffcf6
1 parent 28685d0
commit 77ffcf6
Show file tree

Hide file tree

Showing 8 changed files with 199 additions and 102 deletions.
diff --git a/bamt/config.py b/bamt/config.py
diff --git a/bamt/log.py b/bamt/log.py
@@ -1,32 +1,67 @@
+import logging
 import logging.config
 import os
-import warnings
-
-from bamt.config import config
-
-log_file_path = config.get(
-    "LOG", "log_conf_loc", fallback="log_conf_path is not defined"
-)
-
-if not os.path.isdir(os.path.join(os.path.expanduser("~"), "BAMT")):
-    os.mkdir(os.path.join(os.path.expanduser("~"), "BAMT"))
-
-try:
-    logging.config.fileConfig(log_file_path)
-except BaseException:
-    log_file_path = os.path.join(
-        os.path.dirname(os.path.abspath(__file__)), "logging.conf"
-    )
-    logging.config.fileConfig(log_file_path)
-    warnings.warn(
-        "Reading log path location from config file failed. Default location will be used instead."
-    )
-
-logger_builder = logging.getLogger("builder")
-logger_network = logging.getLogger("network")
-logger_preprocessor = logging.getLogger("preprocessor")
-logger_nodes = logging.getLogger("nodes")
-logger_display = logging.getLogger("display")
-
-logging.captureWarnings(True)
-logger_warnings = logging.getLogger("py.warnings")
+
+
+class BamtLogger:
+    def __init__(self):
+        self.file_handler = False
+        self.enabled = True
+        self.base = os.path.join(os.path.dirname(os.path.abspath(__file__)))
+        self.log_file_path = os.path.join(self.base, "logging.conf")
+        logging.config.fileConfig(self.log_file_path)
+
+        self.loggers = dict(
+            logger_builder=logging.getLogger("builder"),
+            logger_network=logging.getLogger("network"),
+            logger_preprocessor=logging.getLogger("preprocessor"),
+            logger_nodes=logging.getLogger("nodes"),
+            logger_display=logging.getLogger("display"),
+        )
+
+    def has_handler(self, logger, handler_type):
+        """Check if a logger has a handler of a specific type."""
+        return any(isinstance(handler, handler_type) for handler in logger.handlers)
+
+    def remove_handler_type(self, logger, handler_type):
+        """Remove all handlers of a specific type from a logger."""
+        for handler in logger.handlers[:]:
+            if isinstance(handler, handler_type):
+                logger.removeHandler(handler)
+
+    def switch_console_out(self, value: bool):
+        """
+        Turn off console output from logger.
+        """
+        assert isinstance(value, bool)
+        handler_class = logging.StreamHandler if not value else logging.NullHandler
+
+        for logger in self.loggers.values():
+            if self.has_handler(logger, handler_class):
+                self.remove_handler_type(logger, handler_class)
+                logger.addHandler(logging.NullHandler() if not value else logging.root.handlers[0])
+
+    def switch_file_out(self, value: bool, log_file: str):
+        """
+        Send all messages in file
+        """
+        assert isinstance(value, bool)
+
+        for logger in self.loggers.values():
+            if value:
+                file_handler = logging.FileHandler(log_file, mode="a")
+                file_handler.setFormatter(logging.root.handlers[0].formatter)
+                logger.addHandler(file_handler)
+            else:
+                self.remove_handler_type(logger, logging.FileHandler)
+
+
+bamt_logger = BamtLogger()
+
+(
+    logger_builder,
+    logger_network,
+    logger_preprocessor,
+    logger_nodes,
+    logger_display,
+) = list(bamt_logger.loggers.values())
diff --git a/bamt/logging.conf b/bamt/logging.conf
@@ -1,51 +1,44 @@
 [loggers]
-keys=root, preprocessor, builder, nodes, network, display, py.warnings
+keys=root, preprocessor, builder, nodes, network, display
 
 [handlers]
-keys=consoleHandler, fileHandler
+keys=consoleHandler
 
 [formatters]
 keys=simpleFormatter
 
 [logger_root]
 level=INFO
-handlers=fileHandler
+handlers=consoleHandler
 
 [logger_preprocessor]
 level=INFO
 qualname=preprocessor
-handlers=consoleHandler, fileHandler
+handlers=consoleHandler
 propagate=0
 
 [logger_network]
 level=INFO
 qualname=network
-handlers=consoleHandler, fileHandler
+handlers=consoleHandler
 propagate=0
 
 [logger_builder]
 level=INFO
 qualname=builder
-handlers=consoleHandler, fileHandler
+handlers=consoleHandler
 propagate=0
 
 [logger_nodes]
 level=INFO
 qualname=nodes
-handlers=consoleHandler, fileHandler
+handlers=consoleHandler
 propagate=0
 
 [logger_display]
 level=INFO
 qualname=display
-handlers=consoleHandler, fileHandler
-propagate=0
-
-
-[logger_py.warnings]
-level=INFO
-qualname=py.warnings
-handlers=fileHandler
+handlers=consoleHandler
 propagate=0
 
 [handler_consoleHandler]
@@ -54,11 +47,5 @@ level=INFO
 formatter=simpleFormatter
 args=(sys.stdout,)
 
-[handler_fileHandler]
-class=FileHandler
-level=INFO
-formatter=simpleFormatter
-args=(os.path.expanduser("~") + '/BAMT/general.log', 'a')
-
 [formatter_simpleFormatter]
 format=%(asctime)s | %(levelname)-8s | %(filename)s-%(funcName)s-%(lineno)04d | %(message)s
diff --git a/bamt/networks/base.py b/bamt/networks/base.py
@@ -880,8 +880,7 @@ def plot(self, output: str):
         in the parent directory in folder visualization_result.
         output: str name of output file
         """
-        plot_(output, self.nodes, self.edges)
-        return
+        return plot_(output, self.nodes, self.edges)
 
     def markov_blanket(self, node_name, plot_to: Optional[str] = None):
         """

diff --git a/bamt/nodes/base.py b/bamt/nodes/base.py
@@ -1,13 +1,6 @@
-import os
 import pickle
 from typing import Union
 
-from bamt.config import config
-
-STORAGE = config.get(
-    "NODES", "models_storage", fallback="models_storage is not defined"
-)
-
 
 class BaseNode(object):
     """
@@ -61,31 +54,6 @@ def choose_serialization(model) -> Union[str, Exception]:
         except Exception as ex:
             return ex
 
-    @staticmethod
-    def get_path_joblib(node_name: str, specific: str = "") -> str:
-        """
-        Args:
-            node_name: name of node
-            specific: more specific unique name for node.
-            For example, combination.
-
-        Return:
-            Path to save a joblib file.
-        """
-        if not isinstance(specific, str):
-            specific = str(specific)
-
-        index = str(int(os.listdir(STORAGE)[-1]))
-        path_to_check = os.path.join(STORAGE, index, f"{node_name.replace(' ', '_')}")
-
-        if not os.path.isdir(path_to_check):
-            os.makedirs(os.path.join(STORAGE, index, f"{node_name.replace(' ', '_')}"))
-
-        path = os.path.abspath(
-            os.path.join(path_to_check, f"{specific}.joblib.compressed")
-        )
-        return path
-
     @staticmethod
     def get_dist(node_info, pvals):
         pass
diff --git a/docs/source/examples/logger_settings.rst b/docs/source/examples/logger_settings.rst
@@ -0,0 +1,98 @@
+Setting up loggers in BAMT
+===============================
+
+Used imports:
+
+.. code-block:: python
+
+    import pandas as pd
+
+    from sklearn import preprocessing as pp
+
+    import bamt.preprocessors as preprocessors
+    from bamt.networks import ContinuousBN
+
+    from bamt.log import bamt_logger
+
+There are 2 methods to use: ``switch_console_out`` and ``switch_file_out`` of ``bamt_logger``.
+
+By default, bamt will print out messages in console and will not use any log files.
+
+How to turn off/on console output?
+_______________________________
+
+Let's consider this example:
+
+.. code-block:: python
+
+    def learn_bn():
+        hack_data = pd.read_csv("data/real data/hack_processed_with_rf.csv")[
+            [
+                "Tectonic regime",
+                "Period",
+                "Lithology",
+                "Structural setting",
+                "Gross",
+                "Netpay",
+                "Porosity",
+                "Permeability",
+                "Depth",
+            ]
+        ].dropna()
+
+        encoder = pp.LabelEncoder()
+        discretizer = pp.KBinsDiscretizer(n_bins=5, encode="ordinal", strategy="quantile")
+
+        p = preprocessors.Preprocessor([("encoder", encoder), ("discretizer", discretizer)])
+
+        discretized_data, est = p.apply(hack_data)
+
+        bn = ContinuousBN()
+        info = p.info
+
+        bn.add_nodes(info) # here you will get an error
+
+    learn_bn()
+    # The error:
+    # 2023-12-14 16:20:05,010 | ERROR    | base.py-add_nodes-0090 | Continuous BN does not support discrete data
+
+Remove output:
+
+.. code-block:: python
+
+    bamt_logger.switch_console_out(False)
+    learn_bn() # only KeyError from Python
+
+After this you will no longer receive messages from all loggers of BAMT.
+
+To revert changes just use:
+
+.. code-block:: python
+
+    bamt_logger.switch_console_out(True)
+    learn_bn()
+
+    # return
+    # 2023-12-14 16:20:05,010 | ERROR    | base.py-add_nodes-0090 | Continuous BN does not support discrete data
+
+How to turn on/off log files for BAMT?
+______________________________________
+
+In order to redirect errors to log file:
+
+.. code-block:: python
+
+    bamt_logger.switch_file_out(True,
+                                log_file="<absolute/path/to/my_log.log>") # only absolute path
+    learn_bn()
+    # log file
+    # 2023-12-14 16:34:23,414 | ERROR    | base.py-add_nodes-0090 | Continuous BN does not support discrete data
+
+
+To revert this (it will not delete log created before):
+
+.. code-block:: python
+
+    bamt_logger.switch_file_out(False) # only absolute path
+    learn_bn()
+    # log file: no new messages.
diff --git a/docs/source/getting_started/faq.rst b/docs/source/getting_started/faq.rst
@@ -2,3 +2,29 @@
 
 FAQ
 ===
+1. On ``calculate_weights`` I got the following:
+
+.. code-block:: python
+
+    assert np.all([A.dtype == "int" for A in Symbol_matrices])
+    AssertionError
+
+What should I do?
+
+Answer:
+    | Because of not so clear dtypes policies, ``pyitlib`` need all integer columns
+    | as int type (col.dtype must return 'str'). So to fix you can do:
+Instead of:
+
+.. code-block:: python
+
+    bn.calculate_weights(discretized_data)
+
+
+Convert dtypes to intc:
+
+.. code-block:: python
+
+    bn.calculate_weights(discretized_data.astype(np.intc))
+
+
diff --git a/tests/sendingRegressors.py b/tests/sendingRegressors.py
@@ -9,7 +9,7 @@
 from sklearn.tree import DecisionTreeRegressor
 
 import bamt.preprocessors as preprocessors
-from bamt.networks.hybrid_bn import HybridBN
+from bamt.networks import HybridBN
 
 hack_data = pd.read_csv("../data/real data/hack_processed_with_rf.csv")[
     [