From 1a373e2140bdca67a4e49f388162b08a003f9544 Mon Sep 17 00:00:00 2001 From: zoidy Date: Fri, 18 Aug 2023 14:32:28 +0000 Subject: [PATCH 1/8] Remove redundant "Log -" message prefix --- Log.py | 2 +- app.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Log.py b/Log.py index 104e19a..495f4c8 100644 --- a/Log.py +++ b/Log.py @@ -35,7 +35,7 @@ def write_log_in_file(self, type, message, show_in_terminal=False, stop_script=F # Show log in file self.log_config(False) if (show_in_terminal is True): - print(asctime() + ":" + type.upper() + ":Log - " + message) + print(asctime() + ":" + type.upper() + ": " + message) self.message(type, message) if (stop_script is True): exit() diff --git a/app.py b/app.py index dcb06d2..1d82648 100644 --- a/app.py +++ b/app.py @@ -41,12 +41,12 @@ def check_logs_path_access(config_file): logs_access = os.access(log_location, os.W_OK) if (logs_access is False): - print(asctime() + ":ERROR: Log - " + "The logs location specified in the config file could not be reached or read.") + print(asctime() + ":ERROR: " + "The logs location specified in the config file could not be reached or read.") exit() except OSError as error: print(error) - print(asctime() + ":ERROR: Log - " + "The logs location specified in the config file could not be reached or read.") + print(asctime() + ":ERROR: " + "The logs location specified in the config file could not be reached or read.") exit() @@ -56,15 +56,15 @@ def main(): Setting up required variables and conditions. """ global args - print(asctime() + ":Info: Log - ReBACH script has started.") + print(asctime() + ":INFO: ReBACH script has started.") # Check .env file exists. if not args.xfg.is_file(): - print(asctime() + ":ERROR: Log - " + "Configuration file is missing or cannot be read.") + print(asctime() + ":ERROR: " + "Configuration file is missing or cannot be read.") exit() env_file = str(args.xfg) - print(asctime() + ":Info: Log - " + "Env file:" + env_file) - print(asctime() + ":Info: Log - " + "Checking configuration file.") + print(asctime() + ":INFO: " + "Env file:" + env_file) + print(asctime() + ":INFO: " + "Checking configuration file.") config_obj = Config(env_file) figshare_config = config_obj.figshare_config() @@ -80,7 +80,7 @@ def main(): # Check required env variables exist. if (log_location == ""): - print(asctime() + ":ERROR: Log - " + "Logs file path missing in .env.ini file.") + print(asctime() + ":ERROR: " + "Logs file path missing in .env.ini file.") exit() log.write_log_in_file('info', "Logs location is accessible. Logging will now start.", True) From 6c9cbea439ca096db7c7d3d832fefadd751b872f Mon Sep 17 00:00:00 2001 From: zoidy Date: Fri, 18 Aug 2023 14:49:47 +0000 Subject: [PATCH 2/8] Remove the logger name from the messages when logging to file --- Log.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Log.py b/Log.py index 495f4c8..2fc94d4 100644 --- a/Log.py +++ b/Log.py @@ -22,7 +22,7 @@ def log_config(self, in_terminal: bool = False): if (in_terminal): file_path = '' logging.basicConfig(filename=file_path, - format="%(asctime)s:%(levelname)s:%(name)s: %(message)s") + format="%(asctime)s:%(levelname)s: %(message)s") def show_log_in_terminal(self, type, message, stop_script=False): # Show log in terminal @@ -61,3 +61,4 @@ def message(self, type, message): logger.setLevel(logging.ERROR) logger.error(message) del logger + From a956dc44df0abe08384310a862200ce834132a01 Mon Sep 17 00:00:00 2001 From: zoidy Date: Fri, 18 Aug 2023 15:37:57 +0000 Subject: [PATCH 3/8] Normalize timestamp in the console and log file --- Log.py | 3 +-- app.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Log.py b/Log.py index 2fc94d4..f3ed145 100644 --- a/Log.py +++ b/Log.py @@ -1,6 +1,5 @@ from datetime import datetime import logging -from time import asctime from Config import Config @@ -35,7 +34,7 @@ def write_log_in_file(self, type, message, show_in_terminal=False, stop_script=F # Show log in file self.log_config(False) if (show_in_terminal is True): - print(asctime() + ":" + type.upper() + ": " + message) + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":" + type.upper() + ": " + message) self.message(type, message) if (stop_script is True): exit() diff --git a/app.py b/app.py index 1d82648..38a4d52 100644 --- a/app.py +++ b/app.py @@ -3,7 +3,7 @@ from version import __version__, __commit__ from Log import Log from figshare.Article import Article -from time import asctime +from datetime import datetime from Config import Config from figshare.Collection import Collection from pathlib import Path @@ -41,12 +41,12 @@ def check_logs_path_access(config_file): logs_access = os.access(log_location, os.W_OK) if (logs_access is False): - print(asctime() + ":ERROR: " + "The logs location specified in the config file could not be reached or read.") + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " + "The logs location specified in the config file could not be reached or read.") exit() except OSError as error: print(error) - print(asctime() + ":ERROR: " + "The logs location specified in the config file could not be reached or read.") + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " + "The logs location specified in the config file could not be reached or read.") exit() @@ -56,15 +56,15 @@ def main(): Setting up required variables and conditions. """ global args - print(asctime() + ":INFO: ReBACH script has started.") + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":INFO: ReBACH script has started.") # Check .env file exists. if not args.xfg.is_file(): - print(asctime() + ":ERROR: " + "Configuration file is missing or cannot be read.") + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " + "Configuration file is missing or cannot be read.") exit() env_file = str(args.xfg) - print(asctime() + ":INFO: " + "Env file:" + env_file) - print(asctime() + ":INFO: " + "Checking configuration file.") + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":INFO: " + "Env file:" + env_file) + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":INFO: " + "Checking configuration file.") config_obj = Config(env_file) figshare_config = config_obj.figshare_config() @@ -80,7 +80,7 @@ def main(): # Check required env variables exist. if (log_location == ""): - print(asctime() + ":ERROR: " + "Logs file path missing in .env.ini file.") + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " + "Logs file path missing in .env.ini file.") exit() log.write_log_in_file('info', "Logs location is accessible. Logging will now start.", True) From 224e1176821405117864246e6d6ae05b98713932 Mon Sep 17 00:00:00 2001 From: zoidy Date: Fri, 18 Aug 2023 16:07:32 +0000 Subject: [PATCH 4/8] Initialize logs only once --- Log.py | 13 ++++++++----- app.py | 11 +++++------ figshare/Article.py | 4 ++-- figshare/Collection.py | 6 +++--- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Log.py b/Log.py index f3ed145..889cd01 100644 --- a/Log.py +++ b/Log.py @@ -7,20 +7,23 @@ class Log: def __init__(self, config): self.config = config + file_name = "log-" + datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + ".log" - def log_config(self, in_terminal: bool = False): # Setup log configration config_obj = Config(self.config) system_config = config_obj.system_config() log_location = system_config["logs_location"] - file_name = "log-" + datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + ".log" if (log_location[-1] != "/"): log_location = log_location + '/' - file_path = log_location + file_name + self.file_path = log_location + file_name + + def log_config(self, in_terminal: bool = False): if (in_terminal): - file_path = '' - logging.basicConfig(filename=file_path, + f = '' + else: + f = self.file_path + logging.basicConfig(filename=f, force=True, format="%(asctime)s:%(levelname)s: %(message)s") def show_log_in_terminal(self, type, message, stop_script=False): diff --git a/app.py b/app.py index 38a4d52..92a117a 100644 --- a/app.py +++ b/app.py @@ -83,7 +83,7 @@ def main(): print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " + "Logs file path missing in .env.ini file.") exit() - log.write_log_in_file('info', "Logs location is accessible. Logging will now start.", True) + log.write_log_in_file('info', "Logs location is accessible. Logging to file will now start.", True) if (figshare_api_url == "" or figshare_api_token == ""): log.write_log_in_file('error', "Figshare API URL and Token is required.", True, True) @@ -129,18 +129,17 @@ def main(): + " not be reached or read.", True, False) - return env_file + return env_file, log if __name__ == "__main__": get_args() - config_file_path = main() - log = Log(config_file_path) + config_file_path, log = main() log.write_log_in_file('info', "Fetching articles...", True) - article_obj = Article(config_file_path, args.ids) + article_obj = Article(config_file_path, log, args.ids) article_data = article_obj.get_articles() log.write_log_in_file('info', f"Total articles fetched: {len(article_data)}.", @@ -150,7 +149,7 @@ def main(): log.write_log_in_file('info', "Fetching collections...", True) - collection_obj = Collection(config_file_path, args.ids) + collection_obj = Collection(config_file_path, log, args.ids) collection_data = collection_obj.get_collections() log.write_log_in_file('info', f"Total collections fetched: {len(collection_data)}.", diff --git a/figshare/Article.py b/figshare/Article.py index 2aa354d..dba8517 100644 --- a/figshare/Article.py +++ b/figshare/Article.py @@ -23,7 +23,7 @@ class Article: :param config: configuration :param ids: a list of ids to process. If None or an empty list is passed, all will be processed """ - def __init__(self, config, ids): + def __init__(self, config, log, ids): self.config_obj = Config(config) figshare_config = self.config_obj.figshare_config() self.system_config = self.config_obj.system_config() @@ -31,7 +31,7 @@ def __init__(self, config, ids): self.api_token = figshare_config["token"] self.retries = int(figshare_config["retries"]) if figshare_config["retries"] is not None else 3 self.retry_wait = int(figshare_config["retries_wait"]) if figshare_config["retries_wait"] is not None else 10 - self.logs = Log(config) + self.logs = log self.errors = [] self.exclude_dirs = [".DS_Store"] self.total_all_articles_file_size = 0 diff --git a/figshare/Collection.py b/figshare/Collection.py index 94e1727..e649d21 100644 --- a/figshare/Collection.py +++ b/figshare/Collection.py @@ -18,7 +18,7 @@ class Collection: :param config: configuration :param ids: list of ids to process. If None or an empty list is passed, all collections will be processed """ - def __init__(self, config, ids): + def __init__(self, config, log, ids): self.config_obj = Config(config) figshare_config = self.config_obj.figshare_config() self.system_config = self.config_obj.system_config() @@ -27,9 +27,9 @@ def __init__(self, config, ids): self.retries = int(figshare_config["retries"]) if figshare_config["retries"] is not None else 3 self.retry_wait = int(figshare_config["retries_wait"]) if figshare_config["retries_wait"] is not None else 10 self.institution = int(figshare_config["institution"]) - self.logs = Log(config) + self.logs = log self.errors = [] - self.article_obj = Article(config, ids) + self.article_obj = Article(config, log, ids) self.preservation_storage_location = self.system_config["preservation_storage_location"] if self.preservation_storage_location[-1] != "/": self.preservation_storage_location = self.preservation_storage_location + "/" From 7fd8d4be931c28be4860bbc75db097365fb76d26 Mon Sep 17 00:00:00 2001 From: zoidy Date: Fri, 18 Aug 2023 17:22:07 +0000 Subject: [PATCH 5/8] Add color to errors and warnings in the console --- Log.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/Log.py b/Log.py index 889cd01..f39db65 100644 --- a/Log.py +++ b/Log.py @@ -1,5 +1,9 @@ from datetime import datetime import logging +import ctypes +import platform +import sys +import os from Config import Config @@ -18,13 +22,17 @@ def __init__(self, config): log_location = log_location + '/' self.file_path = log_location + file_name + self.ansi_terminal = _check_ansi() + def log_config(self, in_terminal: bool = False): if (in_terminal): f = '' + logging.addLevelName(logging.WARNING, self._format_messagetype_ansi('WARNING')) + logging.addLevelName(logging.ERROR, self._format_messagetype_ansi('ERROR')) else: f = self.file_path logging.basicConfig(filename=f, force=True, - format="%(asctime)s:%(levelname)s: %(message)s") + format='%(asctime)s:%(levelname)s: %(message)s') def show_log_in_terminal(self, type, message, stop_script=False): # Show log in terminal @@ -37,7 +45,7 @@ def write_log_in_file(self, type, message, show_in_terminal=False, stop_script=F # Show log in file self.log_config(False) if (show_in_terminal is True): - print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":" + type.upper() + ": " + message) + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":" + self._format_messagetype_ansi(type.upper()) + ": " + message) self.message(type, message) if (stop_script is True): exit() @@ -64,3 +72,58 @@ def message(self, type, message): logger.error(message) del logger + def _format_messagetype_ansi(self, type): + ''' + Returns a colorized version of the given message type string. If no ANSI support is detected, the same string is returned unchanged. + ''' + if not self.ansi_terminal: + return type + if (type.lower() == 'error'): + return '\033[2;30;41m' + type + '\033[0;0m' + elif (type.lower() == 'warning'): + return '\033[2;31;43m' + type + '\033[0;0m' + elif (type.lower() == 'info'): + return type + elif (type.lower() == 'debug'): + return type + else: + return type + +def _check_ansi(): + ''' + Returns True if the terminal the script is being run in supports ANSI escape sequences + Based on: https://gist.github.com/ssbarnea/1316877 + ''' + for handle in [sys.stdout, sys.stderr]: + if (hasattr(handle, "isatty") and handle.isatty()) or ('TERM' in os.environ and os.environ['TERM']=='ANSI'): + if platform.system()=='Windows' and not ('TERM' in os.environ and os.environ['TERM']=='ANSI'): + if _is_wt(): + # Windows terminal does support ANSI + return True + else: + # Assume the console does not support ANSI + return False + else: + # Assume ANSI available + return True + else: + # no ANSI available + return False + + +def _is_wt(): + ''' + Returns True if the script is run in the Windows Terminal 'wt.exe' + Source: https://github.com/cvzi/AssertWT/blob/3125863ef823d5eaad1bc55155d90d9ca83f4aec/assertwt.py#L74-L88 + ''' + + if platform.system() != 'Windows' or 'idlelib' in sys.modules: + return False + + window = ctypes.windll.kernel32.GetConsoleWindow() + if not window: + return False + ctypes.windll.kernel32.CloseHandle(window) + WM_GETICON = 0x7F + ret = ctypes.windll.user32.SendMessageW(window, WM_GETICON, 0, 0) + return not ret \ No newline at end of file From e8c35c890b3e19ca3cb8a2d4534cb197c8573fbc Mon Sep 17 00:00:00 2001 From: zoidy Date: Fri, 18 Aug 2023 17:27:09 +0000 Subject: [PATCH 6/8] Lint --- Log.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/Log.py b/Log.py index f39db65..6a63fe6 100644 --- a/Log.py +++ b/Log.py @@ -73,21 +73,22 @@ def message(self, type, message): del logger def _format_messagetype_ansi(self, type): - ''' - Returns a colorized version of the given message type string. If no ANSI support is detected, the same string is returned unchanged. - ''' - if not self.ansi_terminal: - return type - if (type.lower() == 'error'): - return '\033[2;30;41m' + type + '\033[0;0m' - elif (type.lower() == 'warning'): - return '\033[2;31;43m' + type + '\033[0;0m' - elif (type.lower() == 'info'): - return type - elif (type.lower() == 'debug'): - return type - else: - return type + ''' + Returns a colorized version of the given message type string. If no ANSI support is detected, the same string is returned unchanged. + ''' + if not self.ansi_terminal: + return type + if (type.lower() == 'error'): + return '\033[2;30;41m' + type + '\033[0;0m' + elif (type.lower() == 'warning'): + return '\033[2;31;43m' + type + '\033[0;0m' + elif (type.lower() == 'info'): + return type + elif (type.lower() == 'debug'): + return type + else: + return type + def _check_ansi(): ''' @@ -95,8 +96,8 @@ def _check_ansi(): Based on: https://gist.github.com/ssbarnea/1316877 ''' for handle in [sys.stdout, sys.stderr]: - if (hasattr(handle, "isatty") and handle.isatty()) or ('TERM' in os.environ and os.environ['TERM']=='ANSI'): - if platform.system()=='Windows' and not ('TERM' in os.environ and os.environ['TERM']=='ANSI'): + if (hasattr(handle, "isatty") and handle.isatty()) or ('TERM' in os.environ and os.environ['TERM'] == 'ANSI'): + if platform.system() == 'Windows' and not ('TERM' in os.environ and os.environ['TERM'] == 'ANSI'): if _is_wt(): # Windows terminal does support ANSI return True @@ -126,4 +127,4 @@ def _is_wt(): ctypes.windll.kernel32.CloseHandle(window) WM_GETICON = 0x7F ret = ctypes.windll.user32.SendMessageW(window, WM_GETICON, 0, 0) - return not ret \ No newline at end of file + return not ret From d3f5c64eee841f92c5bec426762ebab446dc54e4 Mon Sep 17 00:00:00 2001 From: zoidy Date: Fri, 18 Aug 2023 17:29:45 +0000 Subject: [PATCH 7/8] Lint --- app.py | 6 ++++-- figshare/Article.py | 1 - figshare/Collection.py | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 92a117a..b740539 100644 --- a/app.py +++ b/app.py @@ -41,12 +41,14 @@ def check_logs_path_access(config_file): logs_access = os.access(log_location, os.W_OK) if (logs_access is False): - print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " + "The logs location specified in the config file could not be reached or read.") + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " \ + + "The logs location specified in the config file could not be reached or read.") exit() except OSError as error: print(error) - print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " + "The logs location specified in the config file could not be reached or read.") + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " \ + + "The logs location specified in the config file could not be reached or read.") exit() diff --git a/figshare/Article.py b/figshare/Article.py index dba8517..55f6f41 100644 --- a/figshare/Article.py +++ b/figshare/Article.py @@ -6,7 +6,6 @@ import requests import hashlib import re -from Log import Log from Config import Config from figshare.Integration import Integration from slugify import slugify diff --git a/figshare/Collection.py b/figshare/Collection.py index e649d21..67af7a3 100644 --- a/figshare/Collection.py +++ b/figshare/Collection.py @@ -3,7 +3,6 @@ import requests import hashlib import re -from Log import Log from Config import Config from figshare.Article import Article from figshare.Integration import Integration From 744892bea5b703b6315c77021b09ba690ab92470 Mon Sep 17 00:00:00 2001 From: zoidy Date: Fri, 18 Aug 2023 17:31:19 +0000 Subject: [PATCH 8/8] Lint --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index b740539..ec3838f 100644 --- a/app.py +++ b/app.py @@ -41,13 +41,13 @@ def check_logs_path_access(config_file): logs_access = os.access(log_location, os.W_OK) if (logs_access is False): - print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " \ + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " + "The logs location specified in the config file could not be reached or read.") exit() except OSError as error: print(error) - print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " \ + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":ERROR: " + "The logs location specified in the config file could not be reached or read.") exit()