From 268d4bed8876c51c694939c936960478e7deefd4 Mon Sep 17 00:00:00 2001 From: Michael Anderson Date: Tue, 11 Apr 2023 18:41:32 -0700 Subject: [PATCH 1/7] add launch args for log paths; add ros2 param for interval; fix symlinks; add rosdeps Signed-off-by: Michael Anderson --- .gitignore | 136 +++++++++++++++++++++++++-- sim_pblog/.gitignore | 129 ------------------------- sim_pblog/config/sim_pblog.yaml | 2 +- sim_pblog/launch/sim_pblog.launch.py | 24 ++++- sim_pblog/package.xml | 3 + sim_pblog/sim_pblog/sim_pblog.py | 90 ++++++++++-------- 6 files changed, 208 insertions(+), 176 deletions(-) delete mode 100644 sim_pblog/.gitignore diff --git a/.gitignore b/.gitignore index a97d25d..42518ff 100644 --- a/.gitignore +++ b/.gitignore @@ -2,10 +2,132 @@ __pycache__ *.swp *.save* -splinter/include/splinter -splinter/lib -splinter/splinter -splinter/assets -splinter/CREDITS.md -splinter/docs -splinter/README_SPLINTER.md +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/sim_pblog/.gitignore b/sim_pblog/.gitignore deleted file mode 100644 index b6e4761..0000000 --- a/sim_pblog/.gitignore +++ /dev/null @@ -1,129 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ diff --git a/sim_pblog/config/sim_pblog.yaml b/sim_pblog/config/sim_pblog.yaml index 0580a98..0cdf7d9 100644 --- a/sim_pblog/config/sim_pblog.yaml +++ b/sim_pblog/config/sim_pblog.yaml @@ -1,4 +1,4 @@ /sim_pblog: ros__parameters: - foo: 1.0 + logfileinterval_mins: 60 diff --git a/sim_pblog/launch/sim_pblog.launch.py b/sim_pblog/launch/sim_pblog.launch.py index 2260d7b..6fd4194 100644 --- a/sim_pblog/launch/sim_pblog.launch.py +++ b/sim_pblog/launch/sim_pblog.launch.py @@ -15,7 +15,11 @@ import os from ament_index_python.packages import get_package_share_directory + from launch import LaunchDescription +from launch.actions import DeclareLaunchArgument +from launch.substitutions import LaunchConfiguration + from launch_ros.actions import Node @@ -23,20 +27,36 @@ def generate_launch_description(): - ld = LaunchDescription() + loghome_launch_arg = DeclareLaunchArgument( + 'loghome', default_value=[''], + description='root log directory' + ) + + logdir_launch_arg = DeclareLaunchArgument( + 'logdir', default_value=[''], + description='specific log directory in loghome' + ) + config = os.path.join( get_package_share_directory(package_name), 'config', 'sim_pblog.yaml' - ) + ) node = Node( package=package_name, name='sim_pblog', executable='sim_pblog', + arguments=[ + '--loghome', LaunchConfiguration('loghome'), + '--logdir', LaunchConfiguration('logdir') + ], parameters=[config] ) + ld = LaunchDescription() + ld.add_action(loghome_launch_arg) + ld.add_action(logdir_launch_arg) ld.add_action(node) return ld diff --git a/sim_pblog/package.xml b/sim_pblog/package.xml index c98eb0e..c20fb16 100644 --- a/sim_pblog/package.xml +++ b/sim_pblog/package.xml @@ -10,6 +10,9 @@ buoy_interfaces rclpy buoy_api_py + tf_transformations + + ament_copyright ament_flake8 ament_pep257 diff --git a/sim_pblog/sim_pblog/sim_pblog.py b/sim_pblog/sim_pblog/sim_pblog.py index ad5de08..5d74fb9 100755 --- a/sim_pblog/sim_pblog/sim_pblog.py +++ b/sim_pblog/sim_pblog/sim_pblog.py @@ -52,21 +52,28 @@ class WECLogger(Interface): - def __init__(self): - super().__init__('sim_pblog') - self.loghome = '/tmp' # TODO: needs to come from cfg or param - self.logdir = None + def __init__(self, loghome, logdir=None): + super().__init__('sim_pblog', check_for_services=False) + self.start_time = datetime.now() + self.logger_time = self.start_time + + self.loghome = loghome + # Create new log folder at the start of this instance of the logger + self.logdir = self.logdir_setup(logdir) self.logfile = None self.logfilename = None self.logfiletime = datetime.fromtimestamp(0) + + self.logfileinterval_sec = 60 * 60 # in seconds + # Get params from config if available to set self.logfileinterval_sec + self.set_params() + self.pc_header = '' self.bc_header = '' self.xb_header = '' self.sc_header = '' self.tf_header = '' - self.start_time = datetime.now() - self.logger_time = self.start_time - self.logfileinterval = 60 * 60 # in seconds TODO: needs to come from cfg or param + self.logfile_setup() def __del__(self): @@ -77,25 +84,21 @@ def __del__(self): # Example: "2023.03.31T23-59-59.csv" would be created just before midnight on March 31st def logfile_setup(self): - # Create new log folder at the start of this instance of the logger - if (self.logdir is None): - self.logdir = self.logdir_setup() - # close existing log file and zip it shut if (self.logfile is not None): self.close_zip_logfile() # Open new file in logdir using the logger_time (2023.03.23T13.09.54.csv) - self.logfilename = self.logdir \ - + '/' + self.logger_time.strftime('%Y.%m.%dT%I.%M.%S') + '.csv' + csv = self.logger_time.strftime('%Y.%m.%dT%I.%M.%S') + '.csv' + self.logfilename = os.path.join(self.logdir, csv) self.logfile = open(self.logfilename, mode='w', encoding='utf-8') self.write_header() # Point a link called 'latest' to the new directory # Renaming a temporary link works as 'ln -sf' - templink = self.logdir + '/' + '__bogus__' - os.symlink(self.logfilename, templink) - latest = self.logdir + '/' + 'latest' + templink = os.path.join(self.logdir, '__bogus__') + os.symlink(csv, templink) + latest = os.path.join(self.logdir, 'latest_csv') os.rename(templink, latest) self.get_logger().info(f'New log file: {self.logfilename}') @@ -118,7 +121,7 @@ def write_record(self, source_id, data): self.update_logger_time(data) # Create a fresh logfile when interval time has passed - if (self.logger_time > (self.logfiletime + timedelta(seconds=self.logfileinterval))): + if (self.logger_time > (self.logfiletime + timedelta(seconds=self.logfileinterval_sec))): self.logfiletime = self.logger_time self.logfile_setup() @@ -163,25 +166,30 @@ def close_zip_logfile(self): # The system date is used to create a unique directory name for this run # Example: "2023-03-31.005 would be created on the 6th run on March 31st - def logdir_setup(self): + def logdir_setup(self, basename=None): self.get_logger().info(f'Using {self.loghome} as logging home') - # Use logger_time date to create directory name, e.g., 2023-03-23.002 - now = self.logger_time.strftime('%Y-%m-%d') - count = 0 - dirname = self.loghome + '/' + now + '.{nnn:03}'.format(nnn=count) - while (os.path.exists(dirname)): - count = count + 1 - dirname = self.loghome + '/' + now + '.{nnn:03}'.format(nnn=count) - - if (False is os.path.exists(dirname)): + if basename: + dirname = os.path.join(self.loghome, basename) + else: + # Use logger_time date to create directory name, e.g., 2023-03-23.002 + now = self.logger_time.strftime('%Y-%m-%d') + count = 0 + basename = now + '.{nnn:03}'.format(nnn=count) + dirname = os.path.join(self.loghome, basename) + while os.path.exists(dirname): + count = count + 1 + basename = now + '.{nnn:03}'.format(nnn=count) + dirname = os.path.join(self.loghome, basename) + + if not os.path.exists(dirname): os.makedirs(dirname) # Point a link called 'latest' to the new directory # Renaming a temporary link works as 'ln -sf' - templink = self.loghome + '/' + '__templn__' - os.symlink(dirname, templink) - latest = self.loghome + '/' + 'latest' + templink = os.path.join(self.loghome, '__templn__') + os.symlink(basename, templink) + latest = os.path.join(self.loghome, 'latest_csv') os.rename(templink, latest) self.get_logger().info(f'New log directory: {dirname}') @@ -321,7 +329,6 @@ def write_tf(self, data): else: self.logfile.write(',' * self.tf_header.count(',')) - # Delete any unused callback def ahrs_callback(self, data): self.write_record(CrossbowID, data) @@ -337,16 +344,25 @@ def power_callback(self, data): def trefoil_callback(self, data): self.write_record(TrefoilConID, data) - def powerbuoy_callback(self, data): - pass - def set_params(self): - pass + self.declare_parameter('logfileinterval_mins', int(self.logfileinterval_sec / 60)) + self.logfileinterval_sec = \ + 60 * self.get_parameter('logfileinterval_mins').get_parameter_value().integer_value def main(): - rclpy.init() - pblog = WECLogger() + import sys + import argparse + parser = argparse.ArgumentParser() + loghome_arg = parser.add_argument('--loghome', default='/tmp', help='root log directory') + logdir_arg = parser.add_argument('--logdir', help='specific log directory in loghome') + args, extras = parser.parse_known_args() + + rclpy.init(args=extras) + pblog = WECLogger(args.loghome if args.loghome else loghome_arg.default, + args.logdir if args.logdir else logdir_arg.default) + pblog.get_logger().info(f'{args = }') + pblog.get_logger().info(f'{sys.argv = }') rclpy.spin(pblog) rclpy.shutdown() From 42cc1f1f2e85b9eb88be21d30f391c1677162b4e Mon Sep 17 00:00:00 2001 From: andermi Date: Tue, 11 Apr 2023 18:53:23 -0700 Subject: [PATCH 2/7] Update package.xml fix rosdep --- sim_pblog/package.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim_pblog/package.xml b/sim_pblog/package.xml index c20fb16..e43a8db 100644 --- a/sim_pblog/package.xml +++ b/sim_pblog/package.xml @@ -11,7 +11,7 @@ rclpy buoy_api_py tf_transformations - + python3-transforms3d ament_copyright ament_flake8 From e65f11f5f018c9e7d4e4d1be83f907005450e9fa Mon Sep 17 00:00:00 2001 From: Michael Anderson Date: Wed, 12 Apr 2023 11:40:42 -0700 Subject: [PATCH 3/7] fix atexit gzip Signed-off-by: Michael Anderson --- sim_pblog/sim_pblog/sim_pblog.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/sim_pblog/sim_pblog/sim_pblog.py b/sim_pblog/sim_pblog/sim_pblog.py index 5d74fb9..cba5735 100755 --- a/sim_pblog/sim_pblog/sim_pblog.py +++ b/sim_pblog/sim_pblog/sim_pblog.py @@ -14,21 +14,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +import atexit from datetime import datetime, timedelta import gzip import math import os from buoy_api import Interface + from buoy_interfaces.msg import BCRecord from buoy_interfaces.msg import PCRecord from buoy_interfaces.msg import SCRecord from buoy_interfaces.msg import TFRecord from buoy_interfaces.msg import XBRecord + import rclpy from tf_transformations import euler_from_quaternion + # Controller IDs used as the first value in each record BatteryConID = 0 SpringConID = 1 @@ -75,9 +79,7 @@ def __init__(self, loghome, logdir=None): self.tf_header = '' self.logfile_setup() - - def __del__(self): - self.close_zip_logfile() + atexit.register(self.close_zip_logfile) # Create and open a new log file # The system time is used to create a unique CSV log file name @@ -154,13 +156,21 @@ def write_eol(self): # Close the current log file and zip it def close_zip_logfile(self): - if (self.logfile is not None): + if self.logfile is not None and not self.logfile.closed: self.logfile.close() with open(self.logfilename, 'rb') as logf: with gzip.open(f'{self.logfilename}.gz', 'wb') as gzfile: - self.get_logger().info(f'{self.logfilename} -> {self.logfilename}.gz') gzfile.writelines(logf) - os.remove(self.logfilename) + self.get_logger().info(f'{self.logfilename} -> {self.logfilename}.gz') + os.remove(self.logfilename) + + # Point a link called 'latest' to the new directory + # Renaming a temporary link works as 'ln -sf' + csv_gz = os.path.basename(self.logfilename) + '.gz' + templink = os.path.join(self.logdir, '__bogus__') + os.symlink(csv_gz, templink) + latest = os.path.join(self.logdir, 'latest_csv') + os.rename(templink, latest) # Create a new directory for log files created for this instance of logger # The system date is used to create a unique directory name for this run @@ -361,8 +371,6 @@ def main(): rclpy.init(args=extras) pblog = WECLogger(args.loghome if args.loghome else loghome_arg.default, args.logdir if args.logdir else logdir_arg.default) - pblog.get_logger().info(f'{args = }') - pblog.get_logger().info(f'{sys.argv = }') rclpy.spin(pblog) rclpy.shutdown() From 488c9f07046dfb6bd4d0a48d133d1c1905750e50 Mon Sep 17 00:00:00 2001 From: Michael Anderson Date: Wed, 12 Apr 2023 11:42:32 -0700 Subject: [PATCH 4/7] linter Signed-off-by: Michael Anderson --- sim_pblog/sim_pblog/sim_pblog.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sim_pblog/sim_pblog/sim_pblog.py b/sim_pblog/sim_pblog/sim_pblog.py index cba5735..f6ea835 100755 --- a/sim_pblog/sim_pblog/sim_pblog.py +++ b/sim_pblog/sim_pblog/sim_pblog.py @@ -361,7 +361,6 @@ def set_params(self): def main(): - import sys import argparse parser = argparse.ArgumentParser() loghome_arg = parser.add_argument('--loghome', default='/tmp', help='root log directory') From c38c026963a37e861cba9dfcab6fda8af1287267 Mon Sep 17 00:00:00 2001 From: Michael Anderson Date: Wed, 3 May 2023 10:10:42 -0700 Subject: [PATCH 5/7] update default pblog root; stop gzipping at exit; fix gzipping at start Signed-off-by: Michael Anderson --- sim_pblog/launch/sim_pblog.launch.py | 2 +- sim_pblog/sim_pblog/sim_pblog.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/sim_pblog/launch/sim_pblog.launch.py b/sim_pblog/launch/sim_pblog.launch.py index 6fd4194..30ada7e 100644 --- a/sim_pblog/launch/sim_pblog.launch.py +++ b/sim_pblog/launch/sim_pblog.launch.py @@ -28,7 +28,7 @@ def generate_launch_description(): loghome_launch_arg = DeclareLaunchArgument( - 'loghome', default_value=[''], + 'loghome', default_value=['~/.pblogs'], description='root log directory' ) diff --git a/sim_pblog/sim_pblog/sim_pblog.py b/sim_pblog/sim_pblog/sim_pblog.py index f6ea835..b72d6a8 100755 --- a/sim_pblog/sim_pblog/sim_pblog.py +++ b/sim_pblog/sim_pblog/sim_pblog.py @@ -14,11 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import atexit from datetime import datetime, timedelta import gzip import math import os +import sys from buoy_api import Interface @@ -61,7 +61,7 @@ def __init__(self, loghome, logdir=None): self.start_time = datetime.now() self.logger_time = self.start_time - self.loghome = loghome + self.loghome = os.path.expanduser(loghome) # Create new log folder at the start of this instance of the logger self.logdir = self.logdir_setup(logdir) self.logfile = None @@ -78,9 +78,6 @@ def __init__(self, loghome, logdir=None): self.sc_header = '' self.tf_header = '' - self.logfile_setup() - atexit.register(self.close_zip_logfile) - # Create and open a new log file # The system time is used to create a unique CSV log file name # Example: "2023.03.31T23-59-59.csv" would be created just before midnight on March 31st @@ -363,7 +360,7 @@ def set_params(self): def main(): import argparse parser = argparse.ArgumentParser() - loghome_arg = parser.add_argument('--loghome', default='/tmp', help='root log directory') + loghome_arg = parser.add_argument('--loghome', default='~/.pblogs', help='root log directory') logdir_arg = parser.add_argument('--logdir', help='specific log directory in loghome') args, extras = parser.parse_known_args() From f8de7ea457d6d905835b30c9c3278f4418040c5f Mon Sep 17 00:00:00 2001 From: Michael Anderson Date: Wed, 3 May 2023 10:17:15 -0700 Subject: [PATCH 6/7] remove unused import Signed-off-by: Michael Anderson --- sim_pblog/sim_pblog/sim_pblog.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sim_pblog/sim_pblog/sim_pblog.py b/sim_pblog/sim_pblog/sim_pblog.py index b72d6a8..dd1d466 100755 --- a/sim_pblog/sim_pblog/sim_pblog.py +++ b/sim_pblog/sim_pblog/sim_pblog.py @@ -18,7 +18,6 @@ import gzip import math import os -import sys from buoy_api import Interface From 159de2bbe5dd00de6c3f883e7de6b57162c4906e Mon Sep 17 00:00:00 2001 From: Rich Henthorn Date: Wed, 3 May 2023 15:19:59 -0700 Subject: [PATCH 7/7] Correct faulty CrossbowID value from 5 to 3 --- sim_pblog/sim_pblog/sim_pblog.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sim_pblog/sim_pblog/sim_pblog.py b/sim_pblog/sim_pblog/sim_pblog.py index dd1d466..7d790ee 100755 --- a/sim_pblog/sim_pblog/sim_pblog.py +++ b/sim_pblog/sim_pblog/sim_pblog.py @@ -36,8 +36,8 @@ BatteryConID = 0 SpringConID = 1 PowerConID = 2 +CrossbowID = 3 TrefoilConID = 4 -CrossbowID = 5 # WECLogger - duplicate legacy pblog behavior substituting ROS2 topics for CANBus # Description: @@ -94,9 +94,9 @@ def logfile_setup(self): # Point a link called 'latest' to the new directory # Renaming a temporary link works as 'ln -sf' - templink = os.path.join(self.logdir, '__bogus__') + templink = os.path.join(self.logdir, '__templn__') os.symlink(csv, templink) - latest = os.path.join(self.logdir, 'latest_csv') + latest = os.path.join(self.logdir, 'latest') os.rename(templink, latest) self.get_logger().info(f'New log file: {self.logfilename}') @@ -163,9 +163,9 @@ def close_zip_logfile(self): # Point a link called 'latest' to the new directory # Renaming a temporary link works as 'ln -sf' csv_gz = os.path.basename(self.logfilename) + '.gz' - templink = os.path.join(self.logdir, '__bogus__') + templink = os.path.join(self.logdir, '__templn__') os.symlink(csv_gz, templink) - latest = os.path.join(self.logdir, 'latest_csv') + latest = os.path.join(self.logdir, 'latest') os.rename(templink, latest) # Create a new directory for log files created for this instance of logger @@ -191,7 +191,7 @@ def logdir_setup(self, basename=None): if not os.path.exists(dirname): os.makedirs(dirname) - # Point a link called 'latest' to the new directory + # Point a link called 'latest_csv' to the new directory # Renaming a temporary link works as 'ln -sf' templink = os.path.join(self.loghome, '__templn__') os.symlink(basename, templink)