wip: decorator is back on (sadly), started export command refactoring

perun/api/cli.py

@@ -2,8 +2,10 @@
 
 Uses click https://click.palletsprojects.com/en/8.1.x/ to manage complex cmdline configurations.
 """
+import json
+import sys
 from pathlib import Path
-from typing import Optional
+from typing import Dict, List, Optional
 
 import click
 
@@ -79,12 +81,29 @@ def sensors():
             click.echo(f"   {host}: {ranks}")
 
 
+@cli.command()
+def metadata():
+    """Print global metadata dictionaries in json format."""
+    perun = Perun(config)
+
+    hostMD = perun.l_metadata
+    allHostsMD: Optional[List[Dict]] = perun.comm.gather(hostMD, root=0)
+
+    if perun.comm.Get_rank() == 0 and allHostsMD:
+        metadataDict = {}
+        for host, assignedRanks in perun.host_rank.items():
+            metadataDict[host] = allHostsMD[assignedRanks[0]]
+
+        json.dump(metadataDict, sys.stdout, indent=4)
+
+
 @cli.command()
 @click.argument("input_file", type=click.Path(exists=True))
 @click.argument("output_path", type=click.Path(exists=True))
 @click.option(
     "-f" "--format",
     type=click.Choice([format.value for format in IOFormat]),
+    default=None,
 )
 def export(input_file: str, output_path: str, output_format: Optional[str]):
     """Export existing perun output file to another format."""
@@ -99,7 +118,11 @@ def export(input_file: str, output_path: str, output_format: Optional[str]):
         return
 
     inputFormat = IOFormat.fromSuffix(in_file.suffix)
-    out_format = IOFormat(output_format)
+    if output_format:
+        out_format = IOFormat(output_format)
+    else:
+        out_format = IOFormat.fromSuffix(out_path.suffix)
+
     dataNode = importFrom(in_file, inputFormat)
     exportTo(out_path, dataNode, out_format, rawData=True)
 

perun/api/decorator.py

@@ -1,32 +1,48 @@
 """Decorator module."""
 
-# import functools
+import functools
 
-# from perun import config, log
-# from perun.configuration import read_custom_config, read_environ, save_to_config
-# from perun.perun import monitor_application
+from perun import config, log
+from perun.configuration import read_custom_config, read_environ, save_to_config
+from perun.perun import Perun
 
-# def monitor(
-#     configuration: str = "./.perun.ini",
-#     **conf_kwargs,
-# ):
-#     """Decorate function to monitor its energy usage."""
 
-#     def inner_function(func):
-#         @functools.wraps(func)
-#         def func_wrapper(*args, **kwargs):
-#             # Get custom config and kwargs
-#             read_custom_config(None, None, configuration)
-#             for key, value in conf_kwargs.items():
-#                 save_to_config(key, value)
+def monitor(
+    configuration: str = "./.perun.ini",
+    **conf_kwargs,
+):
+    """Decorate function to monitor its energy usage.
 
-#             read_environ()
-#             log.setLevel(f"{config.get('debug', 'log_lvl')}")
+    Parameters
+    ----------
+    configuration : str, optional
+        Path to configuration file, by default "./.perun.ini"
 
-#             func_result = monitor_application(func, args, kwargs)
+    Returns
+    -------
+    _type_
+        Function with perun monitoring enabled
+    """
 
-#             return func_result
+    def inner_function(func):
+        @functools.wraps(func)
+        def func_wrapper(*args, **kwargs):
+            # Get custom config and kwargs
 
-#         return func_wrapper
+            read_environ()
 
-#     return inner_function
+            read_custom_config(None, None, configuration)
+            for key, value in conf_kwargs.items():
+                save_to_config(key, value)
+
+            log.setLevel(f"{config.get('debug', 'log_lvl')}")
+
+            perun = Perun(config)
+
+            func_result = perun.monitor_application(func, args, kwargs)
+
+            return func_result
+
+        return func_wrapper
+
+    return inner_function

perun/backend/util.py

@@ -1,13 +1,13 @@
 """Backend util."""
 import platform
-from typing import Any, Dict, List, Set
+from typing import Any, Dict, Set
 
 from perun import log
 from perun.backend import Backend
 
 
 def getHostMetadata(
-    backends: List[Backend], backendConfig: Dict[str, Set[str]]
+    backends: Dict[str, Backend], backendConfig: Dict[str, Set[str]]
 ) -> Dict[str, Any]:
     """Return dictionary with the full system metadata based on the provided backend configuration.
 
@@ -33,7 +33,7 @@ def getHostMetadata(
                 log.warn(e)
 
     metadata["backends"] = {}
-    for backend in backends:
+    for backend in backends.values():
         if backend.name in backendConfig:
             metadata["backends"][backend.name] = {}
             sensors = backend.getSensors(backendConfig[backend.name])

perun/perun.py

@@ -9,10 +9,11 @@
 from datetime import datetime
 from multiprocessing import Event, Process, Queue
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Set, Tuple, Type
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
 
 from perun import __version__, log
 from perun.backend import Backend, IntelRAPLBackend, NVMLBackend, PSUTILBackend
+from perun.backend.util import getHostMetadata
 from perun.comm import Comm
 from perun.coordination import getGlobalSensorRankConfiguration, getHostRankDict
 from perun.data_model.data import DataNode, NodeType
@@ -41,6 +42,8 @@ def __init__(self, config: ConfigParser) -> None:
         self._l_sensor_config: Optional[Dict[str, Set[str]]] = None
         self._hostname: Optional[str] = None
         self._host_rank: Optional[Dict[str, List[int]]] = None
+        self._metadata: Optional[Dict] = None
+        self._l_metadata: Optional[Dict] = None
 
     def __del__(self):
         """Perun object destructor."""
@@ -148,9 +151,22 @@ def l_sensors_config(self) -> Dict[str, Set[str]]:
 
         return self._l_sensor_config
 
+    @property
+    def l_metadata(self) -> Dict[str, Any]:
+        """Lazy initialization of local metadata dictionary.
+
+        Returns
+        -------
+        Dict[str, Any]
+            Metadata dictionary
+        """
+        if not self._l_metadata:
+            self._l_metadata = getHostMetadata(self.backends, self.l_sensors_config)
+        return self._l_metadata
+
     def monitor_application(
         self,
-        app: Path,
+        app: Union[Callable, Path],
         app_args: tuple = tuple(),
         app_kwargs: dict = dict(),
     ):
@@ -223,7 +239,7 @@ def monitor_application(
 
     def _run_application(
         self,
-        app: Path,
+        app: Union[Callable, Path],
         app_args: tuple = tuple(),
         app_kwargs: dict = dict(),
         record: bool = True,
@@ -261,24 +277,38 @@ def _run_application(
                 start_event.set()
 
             # 3) Start application
-            try:
-                with open(str(app), "r") as scriptFile:
+            if isinstance(app, Path):
+                try:
+                    with open(str(app), "r") as scriptFile:
+                        start_event.wait()
+                        self.comm.barrier()
+                        log.info(f"Rank {self.comm.Get_rank()}: Started App")
+                        run_starttime = datetime.utcnow()
+                        exec(
+                            scriptFile.read(),
+                            {"__name__": "__main__", "__file__": app.name},
+                        )
+                        log.info(f"Rank {self.comm.Get_rank()}: App Stopped")
+                        stop_event.set()
+                except Exception as e:
+                    log.error(
+                        f"Rank {self.comm.Get_rank()}:  Found error on monitored script: {str(app)}"
+                    )
+                    stop_event.set()
+                    raise e
+
+            elif isinstance(app, types.FunctionType):
+                try:
                     start_event.wait()
                     self.comm.barrier()
                     log.info(f"Rank {self.comm.Get_rank()}: Started App")
                     run_starttime = datetime.utcnow()
-                    exec(
-                        scriptFile.read(),
-                        {"__name__": "__main__", "__file__": app.name},
-                    )
-                    log.info(f"Rank {self.comm.Get_rank()}: App Stopped")
+
+                    app_result = app(*app_args, **app_kwargs)
+                    log.info(f"Rank {self.comm.Get_rank()}: Stopped App")
+                except Exception as e:
                     stop_event.set()
-            except Exception as e:
-                log.error(
-                    f"Rank {self.comm.Get_rank()}:  Found error on monitored script: {str(app)}"
-                )
-                stop_event.set()
-                raise e
+                    raise e
 
             # 4) App finished, stop subrocess and get data
             if queue and perunSP:

pyproject.toml

@@ -77,7 +77,7 @@ profile = "black"
 
 [tool.pydocstyle]
 match-dir = 'perun'
-# convention = 'numpy'
+convention = 'numpy'
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]