Python logging for CSLC workflow

src/compass/defaults/s1_cslc_geo.yaml

@@ -98,6 +98,10 @@ runconfig:
               # Enable/disable generation of the topo layers
               geocode_metadata_layers: True
 
+      logging:
+          # Path where to store the log file
+          path: cslc_s1.log
+
       worker:
           # Optional. To prevent downloading DEM or other data (default: False)
           internet_access: False

src/compass/s1_geocode_metadata.py

@@ -7,20 +7,23 @@
 
 import h5py
 import isce3
+
 import journal
 import numpy as np
 from osgeo import gdal
 from scipy.interpolate import InterpolatedUnivariateSpline
 
 from compass import s1_rdr2geo
 from compass.s1_cslc_qa import QualityAssuranceCSLC
+from compass.utils.error_codes import ErrorCode
 from compass.utils.geo_runconfig import GeoRunConfig
 from compass.utils.h5_helpers import (algorithm_metadata_to_h5group,
                                       identity_to_h5group,
                                       init_geocoded_dataset,
                                       metadata_to_h5group, DATA_PATH,
                                       METADATA_PATH, ROOT_PATH)
 from compass.utils.helpers import bursts_grouping_generator, get_module_name
+from compass.utils.logger import Logger
 from compass.utils.yaml_argparse import YamlArgparse
 from compass.utils.radar_grid import get_decimated_rdr_grd
 
@@ -62,7 +65,7 @@ def _fix_layover_shadow_mask(static_layers_dict, h5_root, geo_grid):
                                   data=temp_arr)
 
 
-def run(cfg, burst, fetch_from_scratch=False):
+def run(cfg, fetch_from_scratch=False, logger=None):
     '''
     Geocode metadata layers in single HDF5
 
@@ -74,10 +77,14 @@ def run(cfg, burst, fetch_from_scratch=False):
         Object containing burst parameters needed for geocoding
     fetch_from_scratch: bool
         If True grabs metadata layers from scratch dir
+    logger: Logger
+        Object to handle logging
     '''
     module_name = get_module_name(__file__)
-    info_channel = journal.info(f"{module_name}.run")
-    info_channel.log(f"Starting {module_name} burst")
+    if logger is None:
+        logger = Logger(workflow='CSLC-S1', log_filename=cfg.logging_params.path)
+        logger.info(module_name, ErrorCode.SAS_PROGRAM_STARTING,
+                    f"Starting {module_name} burst")
 
     # Start tracking processing time
     t_start = time.time()
@@ -205,8 +212,8 @@ def run(cfg, burst, fetch_from_scratch=False):
                 cslc_qa.write_qa_dicts_to_json(out_paths.stats_json_path)
 
     dt = str(timedelta(seconds=time.time() - t_start)).split(".")[0]
-    info_channel.log(
-        f"{module_name} burst successfully ran in {dt} (hr:min:sec)")
+    logger.info(module_name, ErrorCode.SAS_PROGRAM_COMPLETED,
+                f"{module_name} burst successfully ran in {dt} (hr:min:sec)")
 
 
 def geocode_luts(geo_burst_h5, burst, cfg, dst_group_path, item_dict,

src/compass/s1_geocode_slc.py

@@ -10,7 +10,6 @@
 import isce3
 # import raster mode geocode_slc. isce3.geocode.geocode_slc is array mode
 from isce3.ext.isce3.geocode import geocode_slc
-import journal
 import numpy as np
 from s1reader.s1_reader import is_eap_correction_necessary
 
@@ -19,6 +18,7 @@
 from compass.s1_cslc_qa import QualityAssuranceCSLC
 from compass.utils.browse_image import make_browse_image
 from compass.utils.elevation_antenna_pattern import apply_eap_correction
+from compass.utils.error_codes import ErrorCode
 from compass.utils.geo_runconfig import GeoRunConfig
 from compass.utils.h5_helpers import (algorithm_metadata_to_h5group,
                                       corrections_to_h5group,
@@ -28,6 +28,7 @@
                                       metadata_to_h5group,
                                       DATA_PATH, METADATA_PATH, ROOT_PATH)
 from compass.utils.helpers import bursts_grouping_generator, get_module_name
+from compass.utils.logger import Logger
 from compass.utils.lut import cumulative_correction_luts
 from compass.utils.yaml_argparse import YamlArgparse
 
@@ -79,8 +80,9 @@ def run(cfg: GeoRunConfig):
         GeoRunConfig object with user runconfig options
     '''
     module_name = get_module_name(__file__)
-    info_channel = journal.info(f"{module_name}.run")
-    info_channel.log(f"Starting {module_name} burst")
+    logger = Logger(workflow='CSLC-S1', log_filename=cfg.logging_params.path)
+    logger.info(module_name, ErrorCode.SAS_PROGRAM_STARTING,
+                f"Starting {module_name} burst")
 
     # Start tracking processing time
     t_start = time.time()
@@ -281,7 +283,10 @@ def run(cfg: GeoRunConfig):
                                                        cfg)
 
     dt = str(timedelta(seconds=time.time() - t_start)).split(".")[0]
-    info_channel.log(f"{module_name} burst successfully ran in {dt} (hr:min:sec)")
+    logger.info(module_name, ErrorCode.SAS_PROGRAM_COMPLETED,
+                f"{module_name} burst successfully ran in {dt} (hr:min:sec)")
+    # Close log file
+    logger.close_log_stream()
 
 
 if __name__ == "__main__":

src/compass/schemas/s1_cslc_geo.yaml

@@ -45,6 +45,9 @@ runconfig:
         # This section includes parameters to tweak the workflow
         processing: include('processing_options', required=False)
 
+        # This section is related to logging
+        logging: include('logging_options', required=False)
+
         # Worker options (e.g. enable/disable GPU processing, select GPU device ID)
         worker: include('worker_options', required=False)
 
@@ -130,6 +133,10 @@ rdr2geo_options:
     # Enable/disable geocoding of the topo layers
     geocode_metadata_layers: bool(required=False)
 
+logging_options:
+  # File path where to store log file
+  path: str(required=False)
+
 worker_options:
     # To prevent downloading DEM / other data automatically. Default True
     internet_access: bool(required=False)

src/compass/utils/error_codes.py

@@ -0,0 +1,101 @@
+"""
+Error codes (from OPERA PGE)
+TO DO: Define some workflow related error codes
+OPERA ADT and OPERA PGE share the same error code?
+"""
+
+from enum import IntEnum, auto, unique
+
+CODES_PER_RANGE = 1000
+"""Number of error codes allocated to each range"""
+
+INFO_RANGE_START = 0
+"""Starting value for the Info code range"""
+
+DEBUG_RANGE_START = INFO_RANGE_START + CODES_PER_RANGE
+"""Starting value for the Debug code range"""
+
+WARNING_RANGE_START = DEBUG_RANGE_START + CODES_PER_RANGE
+"""Starting value for the Warning code range"""
+
+CRITICAL_RANGE_START = WARNING_RANGE_START + CODES_PER_RANGE
+"""Starting value for the Critical code range"""
+
+@unique
+class ErrorCode(IntEnum):
+    """
+    Error codes for OPERA PGEs.
+    Each code is combined with the designated error code offset defined by
+    the RunConfig to determine the final, logged error code.
+    """
+
+    # Info - 0 to 999
+    OVERALL_SUCCESS = INFO_RANGE_START
+    LOG_FILE_CREATED = auto()
+    LOADING_RUN_CONFIG_FILE = auto()
+    VALIDATING_RUN_CONFIG_FILE = auto()
+    LOG_FILE_INIT_COMPLETE = auto()
+    CREATING_WORKING_DIRECTORY = auto()
+    DIRECTORY_SETUP_COMPLETE = auto()
+    MOVING_LOG_FILE = auto()
+    MOVING_OUTPUT_FILE = auto()
+    SUMMARY_STATS_MESSAGE = auto()
+    RUN_CONFIG_FILENAME = auto()
+    PGE_NAME = auto()
+    SCHEMA_FILE = auto()
+    INPUT_FILE = auto()
+    PROCESSING_INPUT_FILE = auto()
+    USING_CONFIG_FILE = auto()
+    CREATED_SAS_CONFIG = auto()
+    CREATING_OUTPUT_FILE = auto()
+    CREATING_CATALOG_METADATA = auto()
+    CREATING_ISO_METADATA = auto()
+    SAS_PROGRAM_STARTING = auto()
+    SAS_PROGRAM_COMPLETED = auto()
+    QA_SAS_PROGRAM_STARTING = auto()
+    QA_SAS_PROGRAM_COMPLETED = auto()
+    QA_SAS_PROGRAM_DISABLED = auto()
+    CLOSING_LOG_FILE = auto()
+
+    # Debug - 1000 – 1999
+    CONFIGURATION_DETAILS = DEBUG_RANGE_START
+    PROCESSING_DETAILS = auto()
+    SAS_EXE_COMMAND_LINE = auto()
+    SAS_QA_COMMAND_LINE = auto()
+
+    # Warning - 2000 – 2999
+    DATE_RANGE_MISSING = WARNING_RANGE_START
+    NO_RENAME_FUNCTION_FOR_EXTENSION = auto()
+    ISO_METADATA_CANT_RENDER_ONE_VARIABLE = auto()
+    LOGGING_REQUESTED_SEVERITY_NOT_FOUND = auto()
+    LOGGING_SOURCE_FILE_DOES_NOT_EXIST = auto()
+    LOGGING_COULD_NOT_INCREMENT_SEVERITY = auto()
+    LOGGING_RESYNC_FAILED = auto()
+
+    # Critical - 3000 to 3999
+    RUN_CONFIG_VALIDATION_FAILED = CRITICAL_RANGE_START
+    DIRECTORY_CREATION_FAILED = auto()
+    SAS_CONFIG_CREATION_FAILED = auto()
+    CATALOG_METADATA_CREATION_FAILED = auto()
+    LOG_FILE_CREATION_FAILED = auto()
+    INPUT_NOT_FOUND = auto()
+    OUTPUT_NOT_FOUND = auto()
+    INVALID_INPUT = auto()
+    INVALID_OUTPUT = auto()
+    INVALID_CATALOG_METADATA = auto()
+    FILE_MOVE_FAILED = auto()
+    FILENAME_VIOLATES_NAMING_CONVENTION = auto()
+    SAS_PROGRAM_NOT_FOUND = auto()
+    SAS_PROGRAM_FAILED = auto()
+    QA_SAS_PROGRAM_NOT_FOUND = auto()
+    QA_SAS_PROGRAM_FAILED = auto()
+    SAS_OUTPUT_FILE_HAS_MISSING_DATA = auto()
+
+    @classmethod
+    def describe(cls):
+        """
+        Provides a listing of the available error codes and their associated
+        integer values.
+        """
+        for name, member in cls.__members__.items():
+            print(f'{name}: {member.value}')

src/compass/utils/geo_runconfig.py

@@ -140,6 +140,16 @@ def lut_params(self) -> dict:
     def quality_assurance_params(self) -> dict:
         return self.groups.quality_assurance
 
+    @property
+    def logging_params(self) -> dict:
+        return self.groups.logging
+
+    @property
+    def output_dir(self) -> str:
+        date_str = self.sensing_start.strftime("%Y%m%d")
+        burst_id = self.burst_id
+        return f'{super().product_path}/{burst_id}/{date_str}'
+
     @property
     def browse_image_params(self) -> dict:
         return self.groups.quality_assurance.browse_image