Dynamic logging configuration

docs/resources/log.config

@@ -1,28 +1,77 @@
-[loggers]
-keys=root
-
-[handlers]
-keys=consoleHandler,errorFileHandler
-
-[formatters]
-keys=baseFormatter
-
-[logger_root]
-level=INFO
-handlers=consoleHandler,errorFileHandler
-
-[handler_consoleHandler]
-class=StreamHandler
-level=DEBUG
-formatter=baseFormatter
-args=(sys.stdout,)
-
-[handler_errorFileHandler]
-class=FileHandler
-level=ERROR
-formatter=baseFormatter
-args=('error_log.txt', 'a')
-
-[formatter_baseFormatter]
-class=nvflare.fuel.utils.log_utils.BaseFormatter
-format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "formatters": {
+        "baseFormatter": {
+            "()": "nvflare.fuel.utils.log_utils.BaseFormatter",
+            "fmt": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        },
+        "colorFormatter": {
+            "()": "nvflare.fuel.utils.log_utils.ColorFormatter",
+            "fmt": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        },
+        "jsonFormatter": {
+            "()": "nvflare.fuel.utils.log_utils.JsonFormatter",
+            "fmt": "%(asctime)s - %(name)s - %(fullName)s - %(levelname)s - %(message)s"
+        }
+    },
+    "filters": {
+        "FLFilter": {
+            "()": "nvflare.fuel.utils.log_utils.LoggerNameFilter",
+            "logger_names": ["custom", "nvflare.app_common", "nvflare.app_opt"]
+        }
+    },
+    "handlers": {
+        "consoleHandler": {
+            "class": "logging.StreamHandler",
+            "level": "DEBUG",
+            "formatter": "colorFormatter",
+            "filters": [],
+            "stream": "ext://sys.stdout"
+        },
+        "logFileHandler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "DEBUG",
+            "formatter": "baseFormatter",
+            "filename": "log.txt",
+            "mode": "a",
+            "maxBytes": 20971520,
+            "backupCount": 10
+        },
+        "errorFileHandler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "ERROR",
+            "formatter": "baseFormatter",
+            "filename": "log_error.txt",
+            "mode": "a",
+            "maxBytes": 20971520,
+            "backupCount": 10
+        },
+        "jsonFileHandler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "DEBUG",
+            "formatter": "jsonFormatter",
+            "filename": "log.json",
+            "mode": "a",
+            "maxBytes": 20971520,
+            "backupCount": 10
+        },
+        "FLFileHandler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "DEBUG",
+            "formatter": "baseFormatter",
+            "filters": ["FLFilter"],
+            "filename": "log_fl.txt",
+            "mode": "a",
+            "maxBytes": 20971520,
+            "backupCount": 10,
+            "delay": true
+        }
+    },
+    "loggers": {
+        "root": {
+            "level": "INFO",
+            "handlers": ["consoleHandler", "logFileHandler", "errorFileHandler", "jsonFileHandler"]
+        }
+    }
+}

nvflare/apis/fl_constant.py

@@ -245,6 +245,8 @@ class AdminCommandNames(object):
     SHELL_TAIL = "tail"
     SHELL_GREP = "grep"
     APP_COMMAND = "app_command"
+    CONFIGURE_JOB_LOG = "configure_job_log"
+    CONFIGURE_SITE_LOG = "configure_site_log"
 
 
 class ServerCommandNames(object):
@@ -263,6 +265,7 @@ class ServerCommandNames(object):
     HANDLE_DEAD_JOB = "handle_dead_job"
     SERVER_STATE = "server_state"
     APP_COMMAND = "app_command"
+    CONFIGURE_JOB_LOG = "configure_job_log"
 
 
 class ServerCommandKey(object):
@@ -382,7 +385,7 @@ class WorkspaceConstants:
     SITE_FOLDER_NAME = "local"
     CUSTOM_FOLDER_NAME = "custom"
 
-    LOGGING_CONFIG = "log.config"
+    LOGGING_CONFIG = "log_config.json"
     DEFAULT_LOGGING_CONFIG = LOGGING_CONFIG + ".default"
     AUDIT_LOG = "audit.log"
     LOG_FILE_NAME = "log.txt"

nvflare/app_common/tie/py_applet.py

@@ -12,14 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import multiprocessing
-import os
 import sys
 import threading
 import time
 from abc import ABC, abstractmethod
 
 from nvflare.apis.workspace import Workspace
-from nvflare.fuel.utils.log_utils import add_log_file_handler, configure_logging
+from nvflare.fuel.utils.log_utils import configure_logging
 from nvflare.security.logging import secure_format_exception, secure_log_traceback
 
 from .applet import Applet
@@ -94,9 +93,7 @@ def start(self, app_ctx: dict):
             if not self.in_process:
                 # enable logging
                 run_dir = self.workspace.get_run_dir(self.job_id)
-                log_file_name = os.path.join(run_dir, "applet_log.txt")
-                configure_logging(self.workspace)
-                add_log_file_handler(log_file_name)
+                configure_logging(self.workspace, dir_path=run_dir, file_prefix="applet")
             self.runner.start(app_ctx)
 
             # Note: run_func does not return until it runs to completion!

nvflare/app_opt/xgboost/histogram_based_v2/adaptors/adaptor.py

@@ -27,7 +27,7 @@
 from nvflare.apis.workspace import Workspace
 from nvflare.app_opt.xgboost.histogram_based_v2.defs import Constant
 from nvflare.app_opt.xgboost.histogram_based_v2.runners.xgb_runner import AppRunner
-from nvflare.fuel.utils.log_utils import add_log_file_handler, configure_logging, get_obj_logger
+from nvflare.fuel.utils.log_utils import configure_logging, get_obj_logger
 from nvflare.fuel.utils.validation_utils import check_object_type
 from nvflare.security.logging import secure_format_exception, secure_log_traceback
 
@@ -65,8 +65,7 @@ def start(self, ctx: dict):
                 run_dir = self.workspace.get_run_dir(self.job_id)
                 log_file_name = os.path.join(run_dir, f"{self.app_name}_log.txt")
                 print(f"XGB Log: {log_file_name}")
-                configure_logging(self.workspace)
-                add_log_file_handler(log_file_name)
+                configure_logging(self.workspace, dir_path=run_dir, file_prefix=self.app_name)
             self.runner.run(ctx)
             self.stopped = True
         except Exception as e: