Skip to content

Commit

Permalink
record accepts argument savefig
Browse files Browse the repository at this point in the history
  • Loading branch information
@ARSadri
ARSadri committed Oct 24, 2024
1 parent b45f346 commit f1e7fa6
Show file tree
Hide file tree
Showing 13 changed files with 348 additions and 206 deletions.
12 changes: 11 additions & 1 deletion HISTORY.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -365,4 +365,14 @@ History
* allow pyrunner to be imported from lognflow itself
* bug fixed in lognflow.plot
* block_runner supports code_block_id to be int, float or string.
* block_runner supports debugging
* block_runner supports debugging

0.12.19 (2024-10-24)
-------------------
* plt_hist2 is easier to use now
* plt_plot supports shapes N, (1, N)
* record accepts argument savefig
* block_runner reads the code before running
* time_tag is True by default again
* removed all imports from lognflow for faster loading
* all tests are passed!
2 changes: 1 addition & 1 deletion lognflow/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

__author__ = 'Alireza Sadri'
__email__ = '[email protected]'
__version__ = '0.12.18'
__version__ = '0.12.19'

from .lognflow import lognflow, getLogger
from .printprogress import printprogress
Expand Down
148 changes: 68 additions & 80 deletions lognflow/lognflow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,39 +35,25 @@
before storing into the directory using log_var(name, var).
"""
import time
import inspect
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.pyplot import imread as mpl_imread
from matplotlib import animation as matplotlib_animation
from pathlib import Path as pathlib_Path
from itertools import product as itertools_product
from sys import platform as sys_platform
from sys import argv as sys_argv
from os import system as os_system
from tempfile import gettempdir
from dataclasses import dataclass
from .logviewer import logviewer
from .utils import (repr_raw,
replace_all,
select_directory,
name_from_file,
is_builtin_collection,
text_to_collection,
dummy_function)
from .plt_utils import (plt_colorbar,
stack_to_frame,
plt_hist,
plt_surface,
plt_imshow_series,
plt_imshow_subplots,
plt_imshow,
plt_scatter3,
plt_plot,
plt_confusion_matrix)
from typing import Union

import time
import atexit
import inspect
from pathlib import Path as pathlib_Path
from sys import platform as sys_platform
from sys import argv as sys_argv
from os import system as os_system
from tempfile import gettempdir
from dataclasses import dataclass
from typing import Union

import numpy as np
import matplotlib.pyplot as plt

from .utils import (repr_raw,
replace_all,
select_directory,
name_from_file,
text_to_collection)
@dataclass
class varinlog:
data_array : np.ndarray
Expand All @@ -76,6 +62,7 @@ class varinlog:
file_start_time : float
suffix : str
log_counter_limit : int
savefig : bool

@dataclass
class textinlog:
Expand Down Expand Up @@ -141,8 +128,8 @@ class lognflow:
:param time_tag:
File names can carry time_tags in time.time() format or indices. This
is pretty much the most fundamental contribution of lognflow beside
carrying the folders and files paths around. By default it is False and
all file names will have time tags if you set it to True. so,
carrying the folders and files paths around. By default it is True and
all file names will not have time tags if you set it to False. so,
you can give time_tag argument for all logger functions, whose
default is this default. It can also be a string: options
are 'index' or 'time_and_index'. If you use index, instead of time
Expand All @@ -156,10 +143,11 @@ def __init__(self,
log_dir_prefix : str = None,
log_dir_suffix : str = None,
exist_ok : bool = True,
time_tag : Union[bool, str] = False,
time_tag : Union[bool, str] = True,
print_text : bool = True,
main_log_name : str = 'log',
log_flush_period : int = 10):
atexit.register(self.flush_all)
self._init_time = time.time()
self.log_dir_prefix = log_dir_prefix
self.log_dir_suffix = log_dir_suffix
Expand Down Expand Up @@ -219,6 +207,7 @@ def __init__(self,
self.enabled = True
self.counted_vars = {}
self.param_name_set = set()
self.close = self.flush_all

#all depricated and will be removed in a few revisions
self.log_text = self.text
Expand Down Expand Up @@ -708,8 +697,8 @@ def _get_log_counter_limit(self, param, log_size_limit):
cnt_limit = int(log_size_limit/(param.size*param.itemsize))
return cnt_limit

def record(self, parameter_name: str, parameter_value,
suffix = None, log_size_limit: int = int(1e+7)):
def record(self, parameter_name: str, parameter_value, flush = False,
suffix = None, log_size_limit: int = int(1e+7), savefig = False):
"""log a numpy array in buffer then dump
It can be the case that we need to take snapshots of a numpy array
over time. The size of the array would not change and this is hoing
Expand Down Expand Up @@ -765,6 +754,8 @@ def record(self, parameter_name: str, parameter_value,
self.record_flush(log_dirnamesuffix)
file_start_time = self.time_stamp
curr_index = 0
elif flush:
self.record_flush(log_dirnamesuffix)

if(curr_index == 0):
data_array = np.zeros((log_counter_limit, ) + parameter_value.shape,
Expand All @@ -773,27 +764,21 @@ def record(self, parameter_name: str, parameter_value,

try:
time_array[curr_index] = self.time_stamp
except:
self.text(
self.log_name,
f'current index {curr_index} cannot be used in the logger')
except: pass
if(parameter_value.shape == data_array[curr_index].shape):
data_array[curr_index] = parameter_value
else:
self.text(
self.log_name,
f'Shape of variable {log_dirnamesuffix} cannot change shape '\
f'from {data_array[curr_index].shape} '\
f'to {parameter_value.shape}. Coppying from the last time.')
data_array[curr_index] = data_array[curr_index - 1]
data_array[curr_index] = data_array[curr_index - 1] * 0 + np.nan

self._vars_dict[log_dirnamesuffix] = varinlog(data_array,
time_array,
curr_index,
file_start_time,
suffix,
log_counter_limit)
log_counter_limit,
savefig)

def record_flush(self, parameter_name: str, suffix: str = None):
def record_flush(self, parameter_name: str, suffix: str = None, savefig = None):
""" Flush the buffered numpy arrays
If you have been using log_ver, this will flush all the buffered
arrays. It is called using log_size_limit for a variable and als
Expand All @@ -814,6 +799,8 @@ def record_flush(self, parameter_name: str, suffix: str = None):
_param_dir = self._get_fpath(param_dir)

_var = self._vars_dict[log_dirnamesuffix]
savefig = _var.savefig if savefig is None else savefig

_var_data_array = _var.data_array[_var.time_array > 0]
_var_time_array = _var.time_array[_var.time_array > 0]
if((_var.suffix == 'npz') | (_var.suffix == 'npy')):
Expand All @@ -826,6 +813,9 @@ def record_flush(self, parameter_name: str, suffix: str = None):
np.savetxt(fpath, _var_time_array)
fpath = _param_dir / f'{param_name}_data_{_var.file_start_time}.txt'
np.savetxt(fpath, _var_data_array)
if savefig:
self.plot(parameter_name, [_var_data_array], '-*',
x_values_list = [_var_time_array])
return fpath

def get_record(self, parameter_name: str, suffix: str = None) -> tuple:
Expand Down Expand Up @@ -998,7 +988,7 @@ def plot(self, parameter_name: str,
"""
if not self.enabled: return
time_tag = self.time_tag if (time_tag is None) else time_tag
from .plt_utils import plt_plot
plt_plot(
parameter_value_list, *plt_plot_args,
x_values_list = x_values_list, fig_ax = fig_ax, **kwargs)
Expand Down Expand Up @@ -1048,6 +1038,7 @@ def hist(self, parameter_name: str,
if not self.enabled: return
time_tag = self.time_tag if (time_tag is None) else time_tag

from .plt_utils import plt_hist
fig, ax = plt_hist(parameter_value_list,
n_bins = n_bins, alpha = alpha,
normalize = normalize,
Expand Down Expand Up @@ -1101,6 +1092,7 @@ def scatter3(self, parameter_name: str,
data_N_by_3 = data_N_by_3.T
self.text(
None, 'lognflow.scatter3> input dataset is transposed.')
from .plt_utils import plt_scatter3
fig_ax_opt_stack = plt_scatter3(data_N_by_3, title = title,
elev_list = elev_list, azim_list = azim_list,
make_animation = make_animation, **kwargs)
Expand Down Expand Up @@ -1138,6 +1130,7 @@ def surface(self, parameter_name: str,
if not self.enabled: return
time_tag = self.time_tag if (time_tag is None) else time_tag

from .plt_utils import plt_surface
fig, ax = plt_surface(parameter_value)

if title is not None:
Expand Down Expand Up @@ -1241,13 +1234,15 @@ def imshow(self,
use_stack_to_frame = True

if(use_stack_to_frame):
from .plt_utils import stack_to_frame
parameter_value = stack_to_frame(
parameter_value, frame_shape = frame_shape,
borders = borders)
if parameter_value is not None:
FLAG_img_ready = True

if(FLAG_img_ready):
from .plt_utils import plt_imshow
plt_imshow(parameter_value,
colorbar = colorbar,
remove_axis_ticks = remove_axis_ticks,
Expand Down Expand Up @@ -1316,6 +1311,7 @@ def imshow_subplots(self,
if not self.enabled: return
time_tag = self.time_tag if (time_tag is None) else time_tag

from .plt_utils import plt_imshow_subplots
fig, ax = plt_imshow_subplots(images = images,
frame_shape = frame_shape,
grid_locations = grid_locations,
Expand Down Expand Up @@ -1394,7 +1390,8 @@ def imshow_series(self,
"""
if not self.enabled: return
time_tag = self.time_tag if (time_tag is None) else time_tag


from .plt_utils import plt_imshow_series
fig, ax = plt_imshow_series(list_of_stacks,
list_of_masks = list_of_masks,
figsize = figsize,
Expand Down Expand Up @@ -1503,7 +1500,7 @@ def log_confusion_matrix(self,
"""
if not self.enabled: return

from .plt_utils import plt_confusion_matrix
plt_confusion_matrix(
cm, target_names=target_names, title=title,
cmap=cmap, figsize=figsize)
Expand Down Expand Up @@ -1544,6 +1541,8 @@ def log_animation(
im = ax.imshow(img, animated=True)
ax.axis('off')
ims.append([im])

from matplotlib import animation as matplotlib_animation
ani = matplotlib_animation.ArtistAnimation(\
fig, ims, interval = interval, blit = blit, repeat_delay = repeat_delay)
try:
Expand Down Expand Up @@ -1572,23 +1571,6 @@ def savez(self, parameter_name: str,
suffix = 'npz',
time_tag = time_tag)

def close(self):
try:
self.flush_all()
except:
print('lognflow: cannot close')

def __call__(self, *args, **kwargs):
"""calling the object
In the case of the following code::
logger = lognflow()
logger('Hello lognflow')
The text (str(...)) will be passed to the main log text file.
"""
fpath = self.text(None, *args, **kwargs)
self.flush_all()
return fpath

#towards supporting all that logging supports
def debug(self, text_to_log):
self.text('debug', text_to_log, time_tag = False)
Expand Down Expand Up @@ -1858,7 +1840,8 @@ def _load(self, var_name, file_index = None,
from torch import load as torch_load
return(torch_load(var_path), var_path)
try: #png
img = mpl_imread(var_path)
from matplotlib.pyplot import imread
img = imread(var_path)
return(img, var_path)
except: pass
try:
Expand Down Expand Up @@ -1948,7 +1931,7 @@ def get_stack_from_files(self,
else:
flist = list(flist)
assert pathlib_Path(flist[0]).is_file(), \
f'File not found: {flist[0]}. You can use logviewer get_flist'
f'File not found: {flist[0]}. You can use get_flist'

if flist:
n_files = len(flist)
Expand All @@ -1959,8 +1942,9 @@ def get_stack_from_files(self,
except: pass
if(read_func is None):
try:
fdata = mpl_imread(flist[0])
read_func = mpl_imread
from matplotlib.pyplot import imread
fdata = imread(flist[0])
read_func = imread
except: pass
try:
read_func(flist[0])
Expand Down Expand Up @@ -2053,12 +2037,16 @@ def replace_time_with_index(self, var_name, verbose = False):
self.text(None, f'To {fpath_new.name}')
flist[fcnt].rename(fpath_new)

def __del__(self):
try:
self.flush_all()
except:
pass

def __call__(self, *args, **kwargs):
"""calling the object
In the case of the following code::
logger = lognflow()
logger('Hello lognflow')
The text (str(...)) will be passed to the main log text file.
"""
fpath = self.text(None, *args, **kwargs)
return fpath

def __repr__(self):
return f'{self.log_dir}'

Expand Down
Loading

0 comments on commit f1e7fa6

Please sign in to comment.