Accumulate logs in a file so historic logs are easier to reference for comparisons (#55)

* Accumulate logs in a file so historic logs are easier to reference for comparisons

Author: AndrewNepogoda

cli/README.md

@@ -114,6 +114,8 @@ data_path="./data"
 | Path to `loader.toml`                                                                                                                                                                   | `--env-config`                           | `False`     | `../config/loader.toml` | `String`                |                                                                                                                                       |
 | When loading each multi-part source, <br/>load all partitions (and shards) in one transaction                                                                                           | `--collapse-partitions-on-load`          | `False`     | `True`                  | `BooleanOptionalAction` | `True` - `--collapse-partitions-on-load`, `False` - `--no-collapse-partitions-on-load`, no argument - default value                   |
 | Logging level for cli                                                                                                                                                                   | `--log-level`                            | `False`     | `INFO`                  | `String`                | `['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']`                                                                                   |
+| Log rotation option. If `date` options is enabled RWM rotates logs each day. If `size` option is enabled RWM rotates log file when it reaches this size                                 | `--log-rotation`                         | `False`     | `date`                  | `String`                | `['date', 'size']`                                                                                                                    |
+| Rotation log file size in Mb. RWM rotates log file when it reaches this size and `--log-rotation` is `size                                                                              | `--log-file-size`                        | `False`     | `5`                     | `Int`                   |                                                                                                                                       |
 | Drop database before workflow run, or not                                                                                                                                               | `--drop-db`                              | `False`     | `False`                 | `BooleanOptionalAction` | `True` - `--drop-db`, `False` - `--no-drop-db`, no argument - default value                                                           |
 | Remove RAI engine and database after run or not                                                                                                                                         | `--cleanup-resources`                    | `False`     | `False`                 | `BooleanOptionalAction` | `True` - `--cleanup-resources`, `False` - `--no-cleanup-resources`, no argument - default value                                       |
 | Remove RAI engine after run or not                                                                                                                                                      | `--cleanup-engine`                       | `False`     | `False`                 | `BooleanOptionalAction` | `True` - `--cleanup-engine`, `False` - `--no-cleanup-engine`, no argument - default value                                             |

cli/args.py

@@ -1,4 +1,5 @@
 from argparse import ArgumentParser, Namespace, BooleanOptionalAction
+from cli.logger import LogRotationOption
 
 
 def parse() -> Namespace:
@@ -84,6 +85,21 @@ def parse() -> Namespace:
         default="INFO",
         type=str
     )
+    parser.add_argument(
+        "--log-rotation",
+        help="Log rotation option",
+        choices=[LogRotationOption.DATE, LogRotationOption.SIZE],
+        required=False,
+        type=LogRotationOption,
+        default=LogRotationOption.DATE
+    )
+    parser.add_argument(
+        "--log-file-size",
+        help="Rotation log file size in Mb. RWM rotates log file when it reaches this size",
+        required=False,
+        type=int,
+        default=5  # 5 Mb
+    )
     parser.add_argument(
         "--drop-db",
         help="Drop RAI database before run, or not",

cli/logger.py

@@ -1,7 +1,17 @@
 import logging
+from logging.handlers import TimedRotatingFileHandler, RotatingFileHandler
+from workflow.common import BaseEnum
 
+log_filename = 'rwm.log'
 
-def configure(level=logging.INFO) -> logging.Logger:
+
+class LogRotationOption(str, BaseEnum):
+    DATE = 'date'
+    SIZE = 'size'
+
+
+def configure(level=logging.INFO, rotation: LogRotationOption = LogRotationOption.DATE,
+              log_file_size: int = 5) -> logging.Logger:
     # override default logging level for azure
     logger = logging.getLogger('azure.core')
     logger.setLevel(logging.ERROR)
@@ -10,8 +20,24 @@ def configure(level=logging.INFO) -> logging.Logger:
 
     logger = logging.getLogger()
     logger.setLevel(level)
-    handler = logging.StreamHandler()
-    formatter = logging.Formatter('[%(asctime)s][%(levelname)s][%(name)s] %(message)s', "%H:%M:%S")
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
+    formatter = logging.Formatter('[%(asctime)s.%(msecs)03d][%(levelname)s][%(name)s] %(message)s', '%Y-%m-%d %H:%M:%S')
+
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
+
+    if rotation == LogRotationOption.SIZE:
+        max_log_size = log_file_size * 1024 * 1024  # covert to Mb
+        size_rotation_handler = RotatingFileHandler(log_filename, maxBytes=max_log_size, backupCount=30,
+                                                    encoding='utf-8')
+        size_rotation_handler.setFormatter(formatter)
+        logger.addHandler(size_rotation_handler)
+        logger.info("Log rotation by size is enabled")
+    elif rotation == LogRotationOption.DATE:
+        time_rotation_handler = TimedRotatingFileHandler(log_filename, when='midnight', interval=1, backupCount=30,
+                                                         encoding='utf-8')
+        time_rotation_handler.setFormatter(formatter)
+        logger.addHandler(time_rotation_handler)
+        logger.info("Log rotation by date is enabled")
+
     return logger.getChild("cli")

cli/runner.py

@@ -16,7 +16,7 @@ def start():
     # parse arguments
     args = cli.args.parse()
     # configure logger
-    logger = cli.logger.configure(logging.getLevelName(args.log_level))
+    logger = cli.logger.configure(logging.getLevelName(args.log_level), args.log_rotation, args.log_file_size)
     try:
         with open(args.env_config, "rb") as fp:
             loader_config = tomli.load(fp)