#925: Fixed error for retrieving compressed cslc files introduced ear…

…lier in this branch
nasa · Aug 1, 2024 · 3d195b5 · 3d195b5
1 parent 8e55663
commit 3d195b5
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 3 deletions.
diff --git a/data_subscriber/asf_cslc_download.py b/data_subscriber/asf_cslc_download.py
@@ -101,6 +101,7 @@ def run_download(self, args, token, es_conn, netloc, username, password, cmr, jo
 
                     try:
                         head_object = boto3.client("s3").head_object(Bucket=bucket, Key=key)
+                        logger.info(f"Adding CSLC file: {p}")
                     except Exception as e:
                         logger.error("Failed when accessing the S3 object:" + p)
                         raise e
@@ -191,7 +192,9 @@ def run_download(self, args, token, es_conn, netloc, username, password, cmr, jo
             raise Exception(f"Failed to get compressed cslc for frame {frame_id} and day index {latest_acq_cycle_index}")
 
         for ccslc in ccslcs:
-            c_cslc_s3paths.extend(ccslc["_source"]["metadata"]["product_s3_paths"])
+            cslc_path = ccslc["_source"]["metadata"]["product_s3_paths"]
+            c_cslc_s3paths.extend(cslc_path)
+            logger.info(f"Adding {cslc_path} to c_cslc_s3paths")
 
         # Now acquire the Ionosphere files for the reference dates of the Compressed CSLC products
         logger.info(f"Downloading Ionosphere files for Compressed CSLCs")

diff --git a/data_subscriber/cslc/cslc_query.py b/data_subscriber/cslc/cslc_query.py
@@ -166,6 +166,7 @@ def determine_download_granules(self, granules):
                     download_granules.append(granule)
 
         for granule in unsubmitted:
+            logger.info(f"Merging in unsubmitted granule {granule['unique_id']}: {granule['granule_id']} for triggering consideration")
             download_batch = by_download_batch_id[granule["download_batch_id"]]
             if granule["unique_id"] not in download_batch:
                 download_batch[granule["unique_id"]] = granule

diff --git a/data_subscriber/cslc_utils.py b/data_subscriber/cslc_utils.py
@@ -283,16 +283,18 @@ def get_dependent_compressed_cslcs(self, frame_id, day_index, eu):
         for mm in range(0, m - 1):  # m parameter is inclusive of the current frame at hand
             for burst_id in self.frame_to_bursts[frame_id].burst_ids:
                 ccslc_m_index = get_dependent_ccslc_index(prev_day_indices, mm, self.k, burst_id)
-                ccslcs = eu.query(
+                ccslc = eu.query(
                     index=_C_CSLC_ES_INDEX_PATTERNS,
                     body={"query": {"bool": {"must": [
                         {"term": {"metadata.ccslc_m_index.keyword": ccslc_m_index}}]}}})
 
                 # Should have exactly one compressed cslc per acq cycle per burst
-                if len(ccslcs) != 1:
+                if len(ccslc) != 1:
                     logger.info("Compressed CSLCs for ccslc_m_index: %s was not found in GRQ ES", ccslc_m_index)
                     return False
 
+                ccslcs.extend(ccslc)
+
         logger.info("All Compresseed CSLSs for frame %s at day index %s found in GRQ ES", frame_id, day_index)
         return ccslcs
 def get_dependent_ccslc_index(prev_day_indices, mm, k, burst_id):