From 39486f582bf149deda5b51316ecb5ed8d8394261 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 1 May 2019 12:05:12 +0200 Subject: [PATCH 1/9] new release cycle --- CHANGES.rst | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index ae2e5d1..aace20b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ Changes ======= +0.0.4 (2019-XX-XX) +------------------ + +* (TBD) + 0.0.3 (2019-05-01) ------------------ diff --git a/setup.py b/setup.py index 617d425..2664382 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ # Bump version HERE! -_version_ = '0.0.3' +_version_ = '0.0.4' # List all versions of Python which are supported From 80fe7b3764a7b78381d8cacfddb25d61fb9dfc90 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 1 May 2019 14:55:19 +0200 Subject: [PATCH 2/9] grammar fix --- docs/library_example.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/library_example.rst b/docs/library_example.rst index 5da02e0..f29cfad 100644 --- a/docs/library_example.rst +++ b/docs/library_example.rst @@ -24,7 +24,7 @@ Create a new directory, for instance in your current working directory, named `` consumer_err_func = demo_err.append ) -You have just stated recording all filesystem events that involve a command containing the string ``kate``. Leave the Python shell and write some stuff into the ``demo_dir`` using ``Kate``, the KDE text editor. Once you are finished, go back to your Python shell and terminate the recording. +You have just started recording all filesystem events that involve a command containing the string ``kate``. Put the Python shell aside and write some stuff into the ``demo_dir`` using ``Kate``, the KDE text editor. Once you are finished, go back to your Python shell and terminate the recording. .. code:: python From 45e784e34fdd7f2c333872e146c7d10ca1d3b97c Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 1 May 2019 15:57:07 +0200 Subject: [PATCH 3/9] logging speedup flag -m added --- CHANGES.rst | 2 +- src/loggedfs/_core/cli.py | 13 ++++++++++--- src/loggedfs/_core/defaults.py | 1 + src/loggedfs/_core/fs.py | 5 +++++ src/loggedfs/_core/out.py | 17 +++++++++++++++++ 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index aace20b..8b1139a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ Changes 0.0.4 (2019-XX-XX) ------------------ -* (TBD) +* FEATURE: New flag ``-m``, explicitly excluding all operations from the log that do not have the potential to change the filesystem. Added for convenience. 0.0.3 (2019-05-01) ------------------ diff --git a/src/loggedfs/_core/cli.py b/src/loggedfs/_core/cli.py index 499168f..6da6580 100644 --- a/src/loggedfs/_core/cli.py +++ b/src/loggedfs/_core/cli.py @@ -82,11 +82,16 @@ help = 'Run in library mode. DO NOT USE THIS FROM THE COMMAND LINE!', hidden = True ) +@click.option( + '-m', '--only-modify-operations', + is_flag = True, + help = 'Exclude logging of all operations that can not cause changes in the filesystem. Convenience flag for accelerated logging.' + ) @click.argument( 'directory', type = click.Path(exists = True, file_okay = False, dir_okay = True, resolve_path = True) ) -def cli_entry(f, p, c, s, l, json, buffers, lib, directory): +def cli_entry(f, p, c, s, l, json, buffers, lib, only_modify_operations, directory): """LoggedFS-python is a transparent fuse-filesystem which allows to log every operation that happens in the backend filesystem. Logs can be written to syslog, to a file, or to the standard output. LoggedFS-python allows to specify an XML @@ -97,7 +102,7 @@ def cli_entry(f, p, c, s, l, json, buffers, lib, directory): loggedfs_factory( directory, - **__process_config__(c, l, s, f, p, json, buffers, lib) + **__process_config__(c, l, s, f, p, json, buffers, lib, only_modify_operations) ) @@ -109,7 +114,8 @@ def __process_config__( fuse_allowother, log_json, log_buffers, - lib_mode + lib_mode, + log_only_modify_operations ): if config_fh is not None: @@ -135,6 +141,7 @@ def __process_config__( 'log_file': log_file, 'log_filter': filter_obj, 'log_json': log_json, + 'log_only_modify_operations': log_only_modify_operations, 'log_printprocessname': log_printprocessname, 'log_syslog': not log_syslog_off } diff --git a/src/loggedfs/_core/defaults.py b/src/loggedfs/_core/defaults.py index 74ca7ca..ef119ed 100644 --- a/src/loggedfs/_core/defaults.py +++ b/src/loggedfs/_core/defaults.py @@ -37,5 +37,6 @@ LOG_BUFFERS_DEFAULT = False LOG_ENABLED_DEFAULT = True LOG_JSON_DEFAULT = False +LOG_ONLYMODIFYOPERATIONS_DEFAULT = False LOG_PRINTPROCESSNAME_DEFAULT = True LOG_SYSLOG_DEFAULT = False diff --git a/src/loggedfs/_core/fs.py b/src/loggedfs/_core/fs.py index 7bae100..706eb3d 100644 --- a/src/loggedfs/_core/fs.py +++ b/src/loggedfs/_core/fs.py @@ -53,6 +53,7 @@ LOG_BUFFERS_DEFAULT, LOG_ENABLED_DEFAULT, LOG_JSON_DEFAULT, + LOG_ONLYMODIFYOPERATIONS_DEFAULT, LOG_PRINTPROCESSNAME_DEFAULT, LOG_SYSLOG_DEFAULT ) @@ -131,6 +132,7 @@ def __init__(self, log_file = None, log_filter = None, log_json = LOG_JSON_DEFAULT, + log_only_modify_operations = LOG_ONLYMODIFYOPERATIONS_DEFAULT, log_printprocessname = LOG_PRINTPROCESSNAME_DEFAULT, log_syslog = LOG_SYSLOG_DEFAULT, **kwargs @@ -169,6 +171,8 @@ def __init__(self, raise TypeError('log_buffers must be of type bool') if not isinstance(lib_mode, bool): raise TypeError('lib_mode must be of type bool') + if not isinstance(log_only_modify_operations, bool): + raise TypeError('log_only_modify_operations must be of type bool') if not isinstance(fuse_foreground, bool): raise TypeError('fuse_foreground must be of type bool') @@ -181,6 +185,7 @@ def __init__(self, self._log_buffers = log_buffers self._log_filter = log_filter self._lib_mode = lib_mode + self._log_only_modify_operations = log_only_modify_operations self._logger = get_logger('LoggedFS-python', log_enabled, log_file, log_syslog, self._log_json) diff --git a/src/loggedfs/_core/out.py b/src/loggedfs/_core/out.py index 32c856f..a64e67f 100644 --- a/src/loggedfs/_core/out.py +++ b/src/loggedfs/_core/out.py @@ -183,6 +183,23 @@ def _log_event_( ret_status, ret_value ): + if self._log_only_modify_operations: + if func.__name__ not in ( + 'chmod', + 'chown', + 'link', + 'mkdir', + 'mknod', + 'rename', + 'rmdir', + 'symlink', + 'truncate', + 'unlink', + 'utimens', + 'write' + ): + return + uid, gid, pid = fuse_get_context() p_cmdname = '' From 5eaae14e8e6851551f277e408cfb75a7be7753af Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 1 May 2019 16:00:09 +0200 Subject: [PATCH 4/9] notifier can use new option --- src/loggedfs/_core/notify.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/loggedfs/_core/notify.py b/src/loggedfs/_core/notify.py index 1e46498..fb72766 100644 --- a/src/loggedfs/_core/notify.py +++ b/src/loggedfs/_core/notify.py @@ -38,7 +38,8 @@ from .defaults import ( FUSE_ALLOWOTHER_DEFAULT, - LOG_BUFFERS_DEFAULT + LOG_BUFFERS_DEFAULT, + LOG_ONLYMODIFYOPERATIONS_DEFAULT ) from .filter import filter_pipeline_class from .ipc import receive, end_of_transmission @@ -81,6 +82,7 @@ def __init__(self, post_exit_func = None, # called on exit log_filter = None, log_buffers = LOG_BUFFERS_DEFAULT, + log_only_modify_operations = LOG_ONLYMODIFYOPERATIONS_DEFAULT, fuse_allowother = FUSE_ALLOWOTHER_DEFAULT, background = False # thread in background ): @@ -122,6 +124,8 @@ def __init__(self, raise TypeError('log_filter must either be None or of type filter_pipeline_class') if not isinstance(log_buffers, bool): raise TypeError('log_buffers must be of type bool') + if not isinstance(log_only_modify_operations, bool): + raise TypeError('log_only_modify_operations must be of type bool') if not isinstance(fuse_allowother, bool): raise TypeError('fuse_allowother must be of type bool') if not isinstance(background, bool): @@ -133,6 +137,7 @@ def __init__(self, self._consumer_err_func = consumer_err_func self._log_filter = log_filter self._log_buffers = log_buffers + self._log_only_modify_operations = log_only_modify_operations self._fuse_allowother = fuse_allowother self._background = background @@ -145,6 +150,8 @@ def __init__(self, ] if self._log_buffers: command.append('-b') # also log read and write buffers + if self._log_only_modify_operations: + command.append('-m') if self._fuse_allowother: command.append('-p') command.append(self._directory) From 3e853a1230ec941ea1036fc2f91e620f3ed3683a Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 1 May 2019 17:48:50 +0200 Subject: [PATCH 5/9] fix: join threads, only started in background --- src/loggedfs/_core/notify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loggedfs/_core/notify.py b/src/loggedfs/_core/notify.py index fb72766..7971591 100644 --- a/src/loggedfs/_core/notify.py +++ b/src/loggedfs/_core/notify.py @@ -205,5 +205,5 @@ def terminate(self): proc = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE) proc.wait() - if not self._background: + if self._background: self._t.join() From b7b1abb59a193303e45aae182f9f3500e5fc88fe Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 1 May 2019 17:51:00 +0200 Subject: [PATCH 6/9] added fix to log --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 8b1139a..7d6aa89 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,7 @@ Changes ------------------ * FEATURE: New flag ``-m``, explicitly excluding all operations from the log that do not have the potential to change the filesystem. Added for convenience. +* FIX: Terminating a notify session stared in the background would not join all relevant threads properly. Terminating a notify session started in the foreground would cause an exception due to it attempting to join non-existing threads. 0.0.3 (2019-05-01) ------------------ From efdc31b981eb7bafef77ebe5f0e10b87a25b3a8d Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 1 May 2019 17:51:25 +0200 Subject: [PATCH 7/9] spelling fix --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7d6aa89..a3f2edf 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,7 +5,7 @@ Changes ------------------ * FEATURE: New flag ``-m``, explicitly excluding all operations from the log that do not have the potential to change the filesystem. Added for convenience. -* FIX: Terminating a notify session stared in the background would not join all relevant threads properly. Terminating a notify session started in the foreground would cause an exception due to it attempting to join non-existing threads. +* FIX: Terminating a notify session started in the background would not join all relevant threads properly. Terminating a notify session started in the foreground would cause an exception due to it attempting to join non-existing threads. 0.0.3 (2019-05-01) ------------------ From a37a9853030dffe69bcc35caa42101dba3bca2c6 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 1 May 2019 20:53:28 +0200 Subject: [PATCH 8/9] added example in jupyter notebook --- docs/library_demo.ipynb | 285 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 docs/library_demo.ipynb diff --git a/docs/library_demo.ipynb b/docs/library_demo.ipynb new file mode 100644 index 0000000..dbaae6f --- /dev/null +++ b/docs/library_demo.ipynb @@ -0,0 +1,285 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "import pandas as pd\n", + "\n", + "import loggedfs # Python pass-through filesystem: pip install loggedfs" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "demo_filter = loggedfs.filter_pipeline_class(include_list = [loggedfs.filter_item_class([loggedfs.filter_field_class(\n", + " name = 'proc_cmd', value = re.compile('.*kate.*').match\n", + " )])]) # filter pipeline for filesystem actions coming from commands containing the string \"kate\" (the KDE text editor)\n", + "\n", + "demo_data, demo_err = [], [] # dump logging data here\n", + "\n", + "demo = loggedfs.loggedfs_notify(\n", + " 'demo_dir', # relative path to mountpoint\n", + " background = True, # log in background thread\n", + " log_filter = demo_filter, # apply filter pipeline\n", + " log_only_modify_operations = True, # only log operations that have the potential to modify the filesystem\n", + " log_buffers = True, # include read and write buffers into log\n", + " consumer_out_func = demo_data.append, # consume logging data\n", + " consumer_err_func = demo_err.append # consume potential errors\n", + " ) # MOUNT (and start logging)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "27" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "demo.terminate() # UMOUNT (and stop logging)\n", + "\n", + "assert isinstance(demo_data[-1], loggedfs.end_of_transmission)\n", + "demo_data = demo_data[:-1]\n", + "\n", + "len(demo_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
actionproc_pidparam_pathparam_old_pathparam_new_pathparam_fipparam_buf_lenparam_offsetparam_modereturnstatus
time
1556736520813215826mknod30977/tmp/ernst/demo_dir/.demo_file.txt.kate-swpNaNNaNNaNNaNNaN33188.0NaNTrue
1556736520814149530chmod30977/tmp/ernst/demo_dir/.demo_file.txt.kate-swpNaNNaNNaNNaNNaN33152.0NaNTrue
1556736524870363873mknod30977/tmp/ernst/demo_dir/demo_file.txt.X30977NaNNaNNaNNaNNaN33152.0NaNTrue
1556736524871304006chmod30977/tmp/ernst/demo_dir/demo_file.txt.X30977NaNNaNNaNNaNNaN33188.0NaNTrue
1556736524872197836rename30977NaN/tmp/ernst/demo_dir/demo_file.txt.X30977/tmp/ernst/demo_dir/demo_file.txtNaNNaNNaNNaNNaNTrue
1556736524872611868unlink30977/tmp/ernst/demo_dir/.demo_file.txt.kate-swpNaNNaNNaNNaNNaNNaNNaNTrue
\n", + "
" + ], + "text/plain": [ + " action proc_pid \\\n", + "time \n", + "1556736520813215826 mknod 30977 \n", + "1556736520814149530 chmod 30977 \n", + "1556736524870363873 mknod 30977 \n", + "1556736524871304006 chmod 30977 \n", + "1556736524872197836 rename 30977 \n", + "1556736524872611868 unlink 30977 \n", + "\n", + " param_path \\\n", + "time \n", + "1556736520813215826 /tmp/ernst/demo_dir/.demo_file.txt.kate-swp \n", + "1556736520814149530 /tmp/ernst/demo_dir/.demo_file.txt.kate-swp \n", + "1556736524870363873 /tmp/ernst/demo_dir/demo_file.txt.X30977 \n", + "1556736524871304006 /tmp/ernst/demo_dir/demo_file.txt.X30977 \n", + "1556736524872197836 NaN \n", + "1556736524872611868 /tmp/ernst/demo_dir/.demo_file.txt.kate-swp \n", + "\n", + " param_old_path \\\n", + "time \n", + "1556736520813215826 NaN \n", + "1556736520814149530 NaN \n", + "1556736524870363873 NaN \n", + "1556736524871304006 NaN \n", + "1556736524872197836 /tmp/ernst/demo_dir/demo_file.txt.X30977 \n", + "1556736524872611868 NaN \n", + "\n", + " param_new_path param_fip \\\n", + "time \n", + "1556736520813215826 NaN NaN \n", + "1556736520814149530 NaN NaN \n", + "1556736524870363873 NaN NaN \n", + "1556736524871304006 NaN NaN \n", + "1556736524872197836 /tmp/ernst/demo_dir/demo_file.txt NaN \n", + "1556736524872611868 NaN NaN \n", + "\n", + " param_buf_len param_offset param_mode return status \n", + "time \n", + "1556736520813215826 NaN NaN 33188.0 NaN True \n", + "1556736520814149530 NaN NaN 33152.0 NaN True \n", + "1556736524870363873 NaN NaN 33152.0 NaN True \n", + "1556736524871304006 NaN NaN 33188.0 NaN True \n", + "1556736524872197836 NaN NaN NaN NaN True \n", + "1556736524872611868 NaN NaN NaN NaN True " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_df = pd.DataFrame.from_records(demo_data, index = 'time')\n", + "\n", + "data_df[data_df['action'] != 'write'][['action', 'proc_pid', 'param_path', 'param_old_path', 'param_new_path', 'param_fip', 'param_buf_len', 'param_offset', 'param_mode', 'return', 'status']]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 668851caf68c8e6675c336e398b16710416ef10b Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 1 May 2019 20:55:07 +0200 Subject: [PATCH 9/9] prepare for release --- CHANGES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a3f2edf..f1ce56f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,11 +1,12 @@ Changes ======= -0.0.4 (2019-XX-XX) +0.0.4 (2019-05-01) ------------------ * FEATURE: New flag ``-m``, explicitly excluding all operations from the log that do not have the potential to change the filesystem. Added for convenience. * FIX: Terminating a notify session started in the background would not join all relevant threads properly. Terminating a notify session started in the foreground would cause an exception due to it attempting to join non-existing threads. +* Added Jupyter Notebook example to documentation. 0.0.3 (2019-05-01) ------------------