loopprocessor is added

.readthedocs.yaml

@@ -0,0 +1,35 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+# build:
+#   os: ubuntu-22.04
+#   tools:
+#     python: "3.7"
+#     # You can also specify other tool versions:
+#     # nodejs: "20"
+#     # rust: "1.70"
+#     # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#    - pdf
+#    - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+# python:
+#    install:
+#    - requirements: docs/requirements.txt

HISTORY.rst

@@ -201,4 +201,11 @@ History
 -----------------
 * get_flist returns whatever search pattern means for .glob
 * plt_tight_layout is removed and replaced by bbox
-* You can get name from file when file is within the log_dir root
+* You can get name from file when file is within the log_dir root
+
+0.10.5 (2023-10-18)
+-----------------
+* Added new files for readthedocs
+* copy() checks for proper use of arguments
+* __call__ returns fpath
+* loopprocessor is added

lognflow/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = 'Alireza Sadri'
 __email__ = '[email protected]'
-__version__ = '0.10.4'
+__version__ = '0.10.5'
 
 from .lognflow import lognflow
 from .logviewer import logviewer
@@ -11,4 +11,5 @@
 from .utils import (
     select_directory, select_file, repr_raw, replace_all, 
     text_to_object, stack_to_frame, stacks_to_frames, ssh_system)
-from .multiprocessor import multiprocessor
+from .multiprocessor import multiprocessor
+from .loopprocessor import loopprocessor

lognflow/lognflow.py

@@ -208,7 +208,7 @@ def name_from_file(self, fpath):
         """
         return name_from_file(self.log_dir_str, fpath)
 
-    def copy(self, parameter_name, source, suffix = None,
+    def copy(self, parameter_name = None, source = None, suffix = None,
              time_tag = False):
         """ copy into a new file
             Given a parameter_name, the second argument will be copied into
@@ -226,14 +226,29 @@ def copy(self, parameter_name, source, suffix = None,
                 their new location.
         """
         if not self.enabled: return
-
+        arg_err_msg = 'when using copy, the first argument is the final name '\
+                      ' after copy is finished. The second argument is ' \
+                      ' the absolute path of source file, str(fpath.absolute())'
+        if parameter_name is not None:
+            assert parameter_name == str(parameter_name), arg_err_msg
+        flist = []
         try:
-            if source.is_file():
-                flist = [source]
+            source_as_fpath = pathlib_Path(source)
+            if source_as_fpath.is_file():
+                flist = [source_as_fpath]
+            else:
+                raise ValueError
         except:
-            flist = self.logged.get_flist(source, suffix)
-        assert flist, 'source could not be found to copy'
+            try:
+                flist = self.logged.get_flist(source, suffix)
+            except Exception as e:
+                print(str(e))
+        assert flist, \
+            'source could not be found to copy. \n' + arg_err_msg
 
+        if parameter_name is None:
+            parameter_name = ''
+
         param_dir, param_name, suffix = self._param_dir_name_suffix(
             parameter_name, suffix)
 
@@ -1059,9 +1074,12 @@ def log_hexbin(self, parameter_name: str, parameter_value,
         else:
             return fig, ax
 
-    def log_imshow(self, parameter_name: str, parameter_value, 
+    def log_imshow(self, 
+                   parameter_name: str, 
+                   parameter_value, 
                    frame_shape : tuple = None,
-                   colorbar = True, remove_axis_ticks = True,
+                   colorbar = True,
+                   remove_axis_ticks = True,
                    image_format='jpeg', dpi=1200, cmap = 'viridis',
                    title = None, time_tag: bool = None, borders = 0, 
                    return_figure = False, **kwargs):
@@ -1234,7 +1252,7 @@ def log_imshow_series(self,
                           list_of_masks = None,
                           figsize_ratio = 1,
                           text_as_colorbar = False,
-                          use_colorbar = False,
+                          colorbar = False,
                           cmap = 'viridis',
                           list_of_titles = None,
                           image_format='jpeg', 
@@ -1275,7 +1293,7 @@ def log_imshow_series(self,
             :param text_as_colorbar: bool
                     if True, max and mean and min of each image will be written
                     on it.
-            :param use_colorbar: bool
+            :param colorbar: bool
                     actual colorbar for each iamge will be shown
             :param time_tag: bool
                     Wheather if the time stamp is in the file name or not.
@@ -1288,7 +1306,7 @@ def log_imshow_series(self,
                                 list_of_masks = list_of_masks,
                                 figsize_ratio = figsize_ratio,
                                 text_as_colorbar = text_as_colorbar,
-                                use_colorbar = use_colorbar,
+                                colorbar = colorbar,
                                 cmap = cmap,
                                 list_of_titles = list_of_titles,
                                 )
@@ -1302,6 +1320,43 @@ def log_imshow_series(self,
         else:
             return fig, ax
 
+    def log_images_in_pdf(self,
+        parameter_name: str, 
+        parameter_value: list,
+        time_tag: bool = None,
+        dpi=1200, 
+        **kwargs):
+
+        if not self.enabled: return
+        time_tag = self.time_tag if (time_tag is None) else time_tag
+
+        param_dir, param_name, suffix = self._param_dir_name_suffix(
+            parameter_name, 'pdf')
+        fpath = self._get_fpath(param_dir, param_name, suffix, time_tag)
+
+        try:
+            from PIL import Image
+        except Eception as e:
+            print('install PIL by: --> pip install Pillow')
+            raise e
+        images = [Image.fromarray(_) for _ in parameter_value]
+        images[0].save(
+            fpath, "PDF" , 
+            resolution=dpi, 
+            save_all=True, 
+            append_images=images[1:],
+            **kwargs)
+
+    def variables_to_pdf(self,
+                         parameter_name: str, 
+                         parameter_value: list,
+                         time_tag: bool = None,
+                         dpi = 1200,
+                         **kwargs):
+        images = self.logged.get_stack_from_names(parameter_value)
+        self.log_images_in_pdf(
+            parameter_name, images, time_tag, dpi, **kwargs)
+
     def log_confusion_matrix(self,
                              parameter_name: str,
                              cm,
@@ -1468,8 +1523,9 @@ def __call__(self, *args, **kwargs):
                 logger('Hello lognflow')
             The text (str(...)) will be passed to the main log text file.
         """
-        self.log_text(None, *args, **kwargs)
+        fpath = self.log_text(None, *args, **kwargs)
         self.flush_all()
+        return fpath
 
     def __del__(self):
         try:

lognflow/logviewer.py

@@ -1,6 +1,6 @@
 import pathlib
 import numpy as np
-from   matplotlib.pyplot import imread
+from   matplotlib.pyplot import imread as mpl_imread
 from   .utils            import replace_all, dummy_function, name_from_file
 
 class logviewer:
@@ -174,7 +174,7 @@ def get_text(self, log_name='main_log', flist = None, suffix = 'txt',
                 txt = txt[0]
             return txt
 
-    def get_single(self, var_name, file_index = -1, 
+    def _get_single(self, var_name, file_index = -1, 
                    suffix = None, read_func = None, verbose = False):
         """ get a single variable
             return the value of a saved variable.
@@ -212,7 +212,7 @@ def get_single(self, var_name, file_index = -1,
                 if verbose:
                     self.logger(f'Loading {var_path}')
                 if read_func is not None:
-                    return read_func(var_path)
+                    return (read_func(var_path), var_path)
                 if(var_path.suffix == '.npz'):
                     buf = np.load(var_path)
                     try: #check if it is made by log_var
@@ -221,39 +221,75 @@ def get_single(self, var_name, file_index = -1,
                         data_array = buf['data_array']
                         data_array = data_array[time_array > 0]
                         time_array = time_array[time_array > 0]
-                        return((time_array, data_array))
+                        return((time_array, data_array), var_path)
                     except:
-                        return(buf)
+                        return(buf, var_path)
                 if(var_path.suffix == '.npy'):
-                    return(np.load(var_path))
+                    return(np.load(var_path), var_path)
                 if(var_path.suffix == '.mat'):
                     from scipy.io import loadmat
-                    return(loadmat(var_path))
+                    return(loadmat(var_path), var_path)
                 if(var_path.suffix == '.dm4'):
                     from hyperspy.api import load as hyperspy_api_load
-                    return hyperspy_api_load(var_path).data
+                    return (hyperspy_api_load(var_path).data, var_path)
                 if((var_path.suffix == '.tif') | (var_path.suffix == '.tiff')):
-                    from tifffile import imread
-                    return(imread(var_path))
+                    from tifffile import imread as tifffile_imread
+                    return(tifffile_imread(var_path), var_path)
                 if(var_path.suffix == '.torch'):      
                     from torch import load as torch_load 
-                    return(torch_load(var_path))
+                    return(torch_load(var_path), var_path)
                 try:
-                    img = imread(var_path)
-                    return(img)
+                    img = mpl_imread(var_path)
+                    return(img, var_path)
                 except:
                     pass
                 # if( (var_path.suffix in ['.txt', '.pdb', '.json', '.fasta'])):
-                #     return var_path.read_text()
-                return var_path.read_text()
+                #     return(var_path.read_text(), var_path)
+                try:
+                    txt = var_path.read_text()
+                    return(txt, var_path)
+                except:
+                    var_path = None
             else:
                 var_path = None
 
         if (var_path is None) & verbose:
             self.logger(f'Looking for {var_name} failed. ' + \
                         f'{var_path} is not in: {self.log_dir}')
+        return None, None
 
-    def get_stack_of_files(self, 
+    def get_single(self, var_name, file_index = -1, 
+                   suffix = None, read_func = None, verbose = False,
+                   return_fpath = False):
+        """ get a single variable
+            return the value of a saved variable.
+
+            Parameters
+            ----------
+            :param var_name:
+                variable name
+            :param file_index:
+                If there are many snapshots of a variable, this input can
+                limit the returned to a set of indices.
+            :param suffix:
+                If there are different suffixes availble for a variable
+                this input needs to be set. npy, npz, mat, and torch are
+                supported.
+            :param read_func:
+                a function that takes the Posix path and returns data
+            .. note::
+                when reading a MATLAB file, the output is a dictionary.
+                Also when reading a npz except if it is made by log_var
+        """
+        get_single_data, fpath = self._get_single(
+            var_name = var_name, file_index = file_index, suffix = suffix, 
+            read_func = read_func, verbose = verbose)   
+        if return_fpath:
+            return get_single_data, fpath
+        else:
+            return get_single_data
+
+    def get_stack_from_files(self, 
         var_name = None, flist = [], suffix = None, read_func = None):
 
         """ Get list or data of all files in a directory
@@ -303,25 +339,52 @@ def get_stack_of_files(self,
                     pass
             if(read_func is None):
                 try:
-                    fdata = imread(flist[0])
-                    read_func = imread
+                    fdata = mpl_imread(flist[0])
+                    read_func = mpl_imread
                 except:
                     pass
             try:
                 read_func(flist[0])
-            except e:
+            except Exception as e:
                 self.logger(f'The data file {flist[0]} could not be opened.'
                             'Please provide a read_function in the input.')
                 raise e
             dataset = [read_func(fpath) for fpath in flist]
             try:
-                dataset = np.array(dataset)
+                dataset_array = np.array(dataset)
             except:
-                pass
-            return(dataset)
+                dataset_array = dataset
+            return(dataset_array)
+
+    def get_stack_from_names(self, 
+             var_names = None, read_func = None, return_flist = False):
+        try:
+            var_names_str = str(var_names)
+        except:
+            pass
+        else:
+            var_names = [var_names]
+        assert var_names == list(var_names), \
+            'input should be a list of variable names'
+        dataset = []
+        flist = []
+        for name in var_names:
+            images_flist = self.get_flist(name)
+            if images_flist:
+                for file_index in range(len(images_flist)):
+                    data, fpath = self.get_single(
+                        name, file_index = file_index,
+                        read_func = read_func, return_fpath = True)
+                    if data is not None:
+                        dataset.append(data)
+                        flist.append(fpath)
+        if return_flist:
+            return dataset, flist
+        else:
+            return dataset
 
     def replace_time_with_index(self, var_name):
-        """ index in file names
+        """ index in file var_names
             lognflow uses time stamps to make new log files for a variable.
             That is done by putting time stamp after the name of the variable.
             This function changes all of the time stamps, sorted ascendingly,
@@ -350,7 +413,8 @@ def replace_time_with_index(self, var_name):
                 fpath_new = flist[fcnt].parent / fname_new
                 # self.log_text(None, f'To {fpath_new.name}')
                 flist[fcnt].rename(fpath_new)
-
+
+
     def __repr__(self):
         return f'{self.log_dir}'