fix s1_geocode_stack.py, undefined variable in s1_rdr2geo.py, uni…

…t tests (#198)

* fix undefined variable in `s1_rdr2geo.py`.

* Add a unit test which runs the static layers script. Add cli command for static layers.

* fix `s1_geocode_stack.py` for changes made to yamale template

* add the static layer test file

* add unit test for geocode stack

* tee pytest output to avoid 10 minute job no-output limit

* point to test burst db

* bump version to 0.4.1

* newline for file end

.circleci/config.yml

@@ -14,7 +14,7 @@ jobs:
       - run:
           name: "Run tests"
           command: |
-            docker run test_image sh -c "cd ~/OPERA/COMPASS && pytest"
+            docker run test_image sh -c "cd ~/OPERA/COMPASS && pytest --capture=tee-sys"
 
 workflows:
   build-workflow:

pyproject.toml

@@ -26,6 +26,7 @@ Homepage = "https://github.com/opera-adt/compass"
 # Entry points for the command line interface
 [project.scripts]
 "s1_cslc.py" = "compass.s1_cslc:main"
+"s1_static_layers.py" = "compass.s1_static_layers:main"
 "s1_geocode_stack.py" = "compass.s1_geocode_stack:main"
 "validate_product.py" = "compass.utils.validate_product:main"
 

src/compass/s1_geocode_stack.py

@@ -67,9 +67,6 @@ def create_parser():
                           help='Sqlite3 database file with burst bounding boxes.')
     optional.add_argument('-nf', '--no-flatten', action='store_true',
                           help='If flag is set, disables topographic phase flattening.')
-    optional.add_argument('-m', '--metadata', action='store_true',
-                          help='If flag is set, generates radar metadata layers for each'
-                               ' burst stack (see rdr2geo processing options)')
     optional.add_argument('-nc', '--no-corrections', action='store_true',
                           help='If flag is set, skip the geocoding LUT corrections.')
     optional.add_argument('--unzipped', action='store_true',
@@ -226,7 +223,7 @@ def get_common_burst_ids(data):
 
 
 def create_runconfig(burst_map_row, dem_file, work_dir, flatten, pol, x_spac,
-                     y_spac, enable_metadata, enable_corrections, burst_db_file):
+                     y_spac, enable_corrections, burst_db_file):
     """
     Create runconfig to process geocoded bursts
 
@@ -246,12 +243,10 @@ def create_runconfig(burst_map_row, dem_file, work_dir, flatten, pol, x_spac,
         Spacing of geocoded burst along X-direction
     y_spac: float
         Spacing of geocoded burst along Y-direction
-    enable_metadata: bool
-        Flag to enable/disable metadata generation for each burst stack.
-    burst_db_file: str
-        Path to burst database file to use for burst bounding boxes.
     enable_corrections: bool
         Flag to enable/disable applying corrections to burst stacks.
+    burst_db_file: str
+        Path to burst database file to use for burst bounding boxes.
 
     Returns
     -------
@@ -289,9 +284,6 @@ def create_runconfig(burst_map_row, dem_file, work_dir, flatten, pol, x_spac,
     geocode['x_posting'] = x_spac
     geocode['y_posting'] = y_spac
 
-    # Metadata generation
-    process['rdr2geo']['enabled'] = enable_metadata
-
     date_str = burst.sensing_start.strftime("%Y%m%d")
     os.makedirs(f'{work_dir}/runconfigs', exist_ok=True)
     runconfig_path = f'{work_dir}/runconfigs/geo_runconfig_{date_str}_{str(burst.burst_id)}.yaml'
@@ -351,7 +343,7 @@ def run(slc_dir, dem_file, burst_id=None, common_bursts_only=False, start_date=N
         end_date=None, exclude_dates=None, orbit_dir=None, work_dir='stack',
         pol='co-pol', x_spac=5, y_spac=10, bbox=None, bbox_epsg=4326,
         output_epsg=None, burst_db_file=DEFAULT_BURST_DB_FILE, flatten=True,
-        enable_metadata=False, enable_corrections=True, using_zipped=True):
+        enable_corrections=True, using_zipped=True):
     """Create runconfigs and runfiles generating geocoded bursts for a static
     stack of Sentinel-1 A/B SAFE files.
 
@@ -395,8 +387,6 @@ def run(slc_dir, dem_file, burst_id=None, common_bursts_only=False, start_date=N
         File path to burst database containing EPSG/extent information.
     flatten: bool
         Enable/disable flattening (removal of the DEM phase) of geocoded burst.
-    enable_metadata: bool
-        Enable/disable generation of metadata files for each burst stack.
     enable_corrections: bool
         Enable/disable generation/usage of correction LUTs during geocoding.
     using_zipped: bool
@@ -450,13 +440,8 @@ def run(slc_dir, dem_file, burst_id=None, common_bursts_only=False, start_date=N
     if burst_id is not None:
         burst_map = prune_dataframe(burst_map, 'burst_id', burst_id)
 
-    # Find the rows which are the first ones to process each burst
-    # Only these rows will be used to generate metadata (if turned on)
-    first_rows = [(burst_map.burst_id == b).idxmax() for b in burst_map.burst_id.unique()]
-
     # Ready to geocode bursts
     for row in burst_map.itertuples():
-        do_metadata = enable_metadata and (row.Index in first_rows)
         runconfig_path = create_runconfig(
             row,
             dem_file=dem_file,
@@ -465,7 +450,6 @@ def run(slc_dir, dem_file, burst_id=None, common_bursts_only=False, start_date=N
             pol=pol,
             x_spac=x_spac,
             y_spac=y_spac,
-            enable_metadata=do_metadata,
             enable_corrections=enable_corrections,
             burst_db_file=burst_db_file,
         )
@@ -503,7 +487,6 @@ def main():
         output_epsg=args.output_epsg,
         burst_db_file=args.burst_db_file,
         flatten=not args.no_flatten,
-        enable_metadata=args.metadata,
         enable_corrections=not args.no_corrections,
         using_zipped=not args.unzipped,
     )

src/compass/s1_rdr2geo.py

@@ -84,7 +84,7 @@ def run(cfg, burst=None, save_in_scratch=False):
 
         # save SLC to ENVI for all bursts
         # run rdr2geo for only 1 burst avoid redundancy
-        burst.slc_to_file(f'{output_path}/{out_paths.fname_pol}.slc')
+        burst.slc_to_file(f'{output_path}/{out_paths.file_name_pol}.slc')
 
         # skip burst if id already rdr2geo processed
         # save id if not processed to avoid rdr2geo reprocessing

src/compass/s1_static_layers.py

@@ -1,14 +1,13 @@
-import journal
 import re
 import time
 
-from datetime import timedelta
+import journal
+
+from compass import s1_geocode_metadata, s1_rdr2geo
 from compass.utils.geo_runconfig import GeoRunConfig
-from compass.utils.yaml_argparse import YamlArgparse
 from compass.utils.helpers import (bursts_grouping_generator, get_module_name,
                                    get_time_delta_str)
-from compass import s1_rdr2geo
-from compass import s1_geocode_metadata
+from compass.utils.yaml_argparse import YamlArgparse
 
 
 def _make_rdr2geo_cfg(yaml_runconfig_str):
@@ -75,13 +74,16 @@ def run(cfg: GeoRunConfig):
     info_channel.log(f"{module_name} burst successfully ran in {dt} (hr:min:sec)")
 
 
-if __name__ == "__main__":
+def main():
+    """Create the CLI and run the static layers workflow"""
     # load arguments from command line
     parser = YamlArgparse()
 
     # Get a runconfig dict from command line arguments
     cfg = GeoRunConfig.load_from_yaml(parser.run_config_path,
                                       workflow_name='s1_cslc_geo')
 
-    # Run geocode burst workflow
     run(cfg)
+
+if __name__ == "__main__":
+    main()

src/compass/version.py

@@ -8,6 +8,7 @@
 # release history
 Tag = collections.namedtuple('Tag', 'version date')
 release_history = (
+    Tag('0.4.1', '2023-08-14'),
     Tag('0.4.0', '2023-07-26'),
     Tag('0.3.1', '2023-06-01'),
     Tag('0.3.0', '2023-05-31'),

tests/test_s1_geocode_stack.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+
+from compass import s1_geocode_stack
+
+
+def test_geocode_slc_run(geocode_slc_params, tmpdir):
+    """
+    Run s1_geocode_slc to ensure it does not crash
+
+    Parameters
+    ----------
+    geocode_slc_params: SimpleNamespace
+        SimpleNamespace containing geocode SLC unit test parameters
+    """
+    data_dir = Path(geocode_slc_params.gslc_cfg_path).parent
+
+    with tmpdir.as_cwd():
+        s1_geocode_stack.run(
+            slc_dir=data_dir,
+            dem_file=data_dir / "test_dem.tiff",
+            orbit_dir=data_dir / "orbits",
+            burst_db_file=data_dir / "test_burst_map.sqlite3",
+        )

tests/test_s1_static_layers.py

@@ -0,0 +1,20 @@
+from compass import s1_static_layers
+from compass.utils.geo_runconfig import GeoRunConfig
+
+
+def test_geocode_slc_run(geocode_slc_params):
+    """
+    Run s1_geocode_slc to ensure it does not crash
+
+    Parameters
+    ----------
+    geocode_slc_params: SimpleNamespace
+        SimpleNamespace containing geocode SLC unit test parameters
+    """
+    # load yaml to cfg
+    cfg = GeoRunConfig.load_from_yaml(
+        geocode_slc_params.gslc_cfg_path, workflow_name="s1_cslc_geo"
+    )
+
+    # pass cfg to s1_geocode_slc
+    s1_static_layers.run(cfg)