Adding logger in global level.

pynxtools/dataconverter/convert.py

@@ -33,19 +33,14 @@
 from pynxtools.dataconverter.writer import Writer
 from pynxtools.dataconverter.template import Template
 from pynxtools.nexus import nexus
+from pynxtools.dataconverter.logger import logger as pynx_logger
 
 if sys.version_info >= (3, 10):
     from importlib.metadata import entry_points
 else:
     from importlib_metadata import entry_points
 
 
-logger = logging.getLogger(__name__)  # pylint: disable=C0103
-UNDOCUMENTED = 9
-logger.setLevel(logging.INFO)
-logger.addHandler(logging.StreamHandler(sys.stdout))
-
-
 def get_reader(reader_name) -> BaseReader:
     """Helper function to get the reader object from it's given name"""
     path_prefix = f"{os.path.dirname(__file__)}{os.sep}" if os.path.dirname(__file__) else ""
@@ -123,6 +118,7 @@ def get_nxdl_root_and_path(nxdl: str):
 def transfer_data_into_template(input_file,
                                 reader, nxdl_name,
                                 nxdl_root: Optional[ET.Element] = None,
+                                logger: logging.Logger = pynx_logger,
                                 **kwargs):
     """Transfer parse and merged data from input experimental file, config file and eln.
 
@@ -139,6 +135,8 @@ def transfer_data_into_template(input_file,
         Root name of nxdl file, e.g. NXmpes from NXmpes.nxdl.xml
     nxdl_root : ET.element
         Root element of nxdl file, otherwise provide nxdl_name
+    logger: looging.Logger
+        Logger to get log massages.
 
     Returns
     -------
@@ -156,9 +154,8 @@ def transfer_data_into_template(input_file,
         input_file = (input_file,)
 
     bulletpoint = "\n\u2022 "
-    logger.info("Using %s reader to convert the given files: %s ",
-                reader,
-                bulletpoint.join((" ", *input_file)))
+    logger.info("Using %s reader reader to convert the given files: %s ",
+                reader, bulletpoint.join((' ', *input_file)))
 
     data_reader = get_reader(reader)
     if not (nxdl_name in data_reader.supported_nxdls or "*" in data_reader.supported_nxdls):
@@ -169,7 +166,7 @@ def transfer_data_into_template(input_file,
         file_paths=input_file,
         **kwargs
     )
-    helpers.validate_data_dict(template, data, nxdl_root)
+    helpers.validate_data_dict(template, data, nxdl_root, logger=logger)
     return data
 
 
@@ -181,6 +178,7 @@ def convert(input_file: Tuple[str, ...],
             generate_template: bool = False,
             fair: bool = False,
             undocumented: bool = False,
+            logger: logging.Logger = pynx_logger,
             **kwargs):
     """The conversion routine that takes the input parameters and calls the necessary functions.
 
@@ -201,14 +199,15 @@ def convert(input_file: Tuple[str, ...],
         in the template.
     undocumented : bool, default False
         If True, an undocumented warning is given.
+    logger: looging.Logger
+        Logger to get log massages.
 
     Returns
     -------
     None.
     """
 
     nxdl_root, nxdl_f_path = get_nxdl_root_and_path(nxdl)
-
     if generate_template:
         template = Template()
         helpers.generate_template_from_nxdl(nxdl_root, template)
@@ -217,25 +216,24 @@ def convert(input_file: Tuple[str, ...],
 
     data = transfer_data_into_template(input_file=input_file, reader=reader,
                                        nxdl_name=nxdl, nxdl_root=nxdl_root,
-                                       **kwargs)
-    if undocumented:
-        logger.setLevel(UNDOCUMENTED)
+                                       logger=logger, **kwargs)
+
     if fair and data.undocumented.keys():
         logger.warning("There are undocumented paths in the template. This is not acceptable!")
         return
-
-    for path in data.undocumented.keys():
-        if "/@default" in path:
-            continue
-        logger.log(
-            UNDOCUMENTED,
-            "The path, %s, is being written but has no documentation.",
-            path
-        )
-    helpers.add_default_root_attributes(data=data, filename=os.path.basename(output))
+    if undocumented:
+        for path in data.undocumented.keys():
+            if "/@default" in path:
+                continue
+            logger.info(
+                "NO DOCUMENTATION: The path, %s, is being written but has no documentation.",
+                path)
+
+    helpers.add_default_root_attributes(data=data, filename=os.path.basename(output),
+                                        logger=logger)
     Writer(data=data, nxdl_f_path=nxdl_f_path, output_path=output).write()
 
-    logger.info("The output file generated: %s", output)
+    logger.info("The output file generated: %s ", output)
 
 
 def parse_params_file(params_file):

pynxtools/dataconverter/helpers.py

@@ -22,7 +22,6 @@
 import re
 import xml.etree.ElementTree as ET
 from datetime import datetime, timezone
-import logging
 import json
 
 import numpy as np
@@ -32,8 +31,7 @@
 from pynxtools import get_nexus_version, get_nexus_version_hash
 from pynxtools.nexus import nexus
 from pynxtools.nexus.nexus import NxdlAttributeError
-
-logger = logging.getLogger(__name__)
+from pynxtools.dataconverter.logger import logger as pynx_logger
 
 
 def is_a_lone_group(xml_element) -> bool:
@@ -447,7 +445,9 @@ def does_group_exist(path_to_group, data):
     return False
 
 
-def ensure_all_required_fields_exist(template, data, nxdl_root):
+# pylint: disable=W1203
+def ensure_all_required_fields_exist(template, data,
+                                     nxdl_root, logger=pynx_logger):
     """Checks whether all the required fields are in the returned data object."""
     for path in template["required"]:
         entry_name = get_name_from_data_dict_entry(path[path.rindex('/') + 1:])
@@ -461,15 +461,16 @@ def ensure_all_required_fields_exist(template, data, nxdl_root):
             opt_parent = check_for_optional_parent(path, nxdl_root)
             if opt_parent != "<<NOT_FOUND>>":
                 if does_group_exist(opt_parent, data) and not does_group_exist(renamed_path, data):
-                    raise ValueError(f"The required group, {path}, hasn't been supplied"
-                                     f" while its optional parent, {path}, is supplied.")
+                    logger.warning("The required group, %s, hasn't been supplied"
+                                   " while its optional parent, %s, is supplied.", path,
+                                   opt_parent)
                 continue
             if not does_group_exist(renamed_path, data):
                 raise ValueError(f"The required group, {path}, hasn't been supplied.")
             continue
         if not is_path_in_data_dict or data[renamed_path] is None:
-            raise ValueError(f"The data entry corresponding to {path} is required "
-                             f"and hasn't been supplied by the reader.")
+            logger.warning("The data entry corresponding to %s is required "
+                           "and hasn't been supplied by the reader.", path)
 
 
 def try_undocumented(data, nxdl_root: ET.Element):
@@ -498,7 +499,8 @@ def try_undocumented(data, nxdl_root: ET.Element):
             pass
 
 
-def validate_data_dict(template, data, nxdl_root: ET.Element):
+def validate_data_dict(template, data,
+                       nxdl_root: ET.Element, logger=pynx_logger):
     """Checks whether all the required paths from the template are returned in data dict."""
     assert nxdl_root is not None, "The NXDL file hasn't been loaded."
 
@@ -507,7 +509,7 @@ def validate_data_dict(template, data, nxdl_root: ET.Element):
     nxdl_path_to_elm: dict = {}
 
     # Make sure all required fields exist.
-    ensure_all_required_fields_exist(template, data, nxdl_root)
+    ensure_all_required_fields_exist(template, data, nxdl_root, logger)
     try_undocumented(data, nxdl_root)
 
     for path in data.get_documented().keys():
@@ -590,7 +592,7 @@ def convert_to_hill(atoms_typ):
     return atom_list + list(atoms_typ)
 
 
-def add_default_root_attributes(data, filename):
+def add_default_root_attributes(data, filename, logger=pynx_logger):
     """
     Takes a dict/Template and adds NXroot fields/attributes that are inherently available
     """

pynxtools/dataconverter/logger.py

@@ -0,0 +1,25 @@
+"""Logger for pynxtools"""
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+
+logger = logging.getLogger("pynxtools")
+
+# Lowest level log allows to other levels erros, crittical, info and debug
+logger.setLevel(logging.DEBUG)

tests/dataconverter/test_helpers.py

@@ -25,6 +25,7 @@
 import numpy as np
 
 from pynxtools.dataconverter import helpers
+from pynxtools.dataconverter.logger import logger as pynx_logger
 from pynxtools.dataconverter.template import Template
 
 
@@ -189,6 +190,7 @@ def fixture_filled_test_data(template, tmp_path):
 TEMPLATE["optional"]["/@default"] = "Some NXroot attribute"
 
 
+# pylint: disable=too-many-arguments
 @pytest.mark.parametrize("data_dict,error_message", [
     pytest.param(
         alter_dict(TEMPLATE, "/ENTRY[my_entry]/NXODD_name/int_value", "not_a_num"),
@@ -300,8 +302,7 @@ def fixture_filled_test_data(template, tmp_path):
                          "required"
                          ),
         ("The required group, /ENTRY[entry]/optional_parent/req_group_in_opt_group, hasn't been "
-         "supplied while its optional parent, /ENTRY[entry]/optional_parent/"
-         "req_group_in_opt_group, is supplied."),
+         "supplied while its optional parent, /ENTRY[entry]/optional_parent, is supplied."),
         id="req-group-in-opt-parent-removed"
     ),
     pytest.param(
@@ -310,7 +311,7 @@ def fixture_filled_test_data(template, tmp_path):
         id="opt-group-completely-removed"
     ),
 ])
-def test_validate_data_dict(data_dict, error_message, template, nxdl_root, request):
+def test_validate_data_dict(caplog, data_dict, error_message, template, nxdl_root, request):
     """Unit test for the data validation routine"""
     if request.node.callspec.id in ("valid-data-dict",
                                     "lists",
@@ -322,10 +323,17 @@ def test_validate_data_dict(data_dict, error_message, template, nxdl_root, reque
                                     "link-dict-instead-of-bool",
                                     "allow-required-and-empty-group",
                                     "opt-group-completely-removed"):
-        helpers.validate_data_dict(template, data_dict, nxdl_root)
+        helpers.validate_data_dict(template, data_dict, nxdl_root, logger=pynx_logger)
+    # Missing required fields
+    elif request.node.callspec.id in ("empty-required-field",
+                                      "req-group-in-opt-parent-removed"
+                                      ):
+        captured_logs = caplog.records
+        helpers.validate_data_dict(template, data_dict, nxdl_root, pynx_logger)
+        assert any(error_message in rec.message for rec in captured_logs)
     else:
         with pytest.raises(Exception) as execinfo:
-            helpers.validate_data_dict(template, data_dict, nxdl_root)
+            helpers.validate_data_dict(template, data_dict, nxdl_root, pynx_logger)
         assert (error_message) == str(execinfo.value)