apply flat sensitivity on the signal before processing

justfile

@@ -138,15 +138,17 @@ main-cloud-mars-multiple-days year="2022" month="9" *days="5 7 8 9":
     done
     wait
 
+#    export MAX_SEGMENTS=60
+#    export ASSUME_DOWNLOADED_FILES=yes
 # chumash basic test for cloud processing
-main-cloud-chumash-basic-test max_segments="60" date="20230101":
+main-cloud-chumash-basic-test date="20230108":
     #!/usr/bin/env bash
     export DATE={{date}}
     export S3_JSON_BUCKET_PREFIX="s3://pacific-sound-metadata/ch01"
-    export SENSITIVITY_FLAT_VALUE=176
+    export SENSITIVITY_FLAT_VALUE=176.1
+    export OUTPUT_PREFIX="CH01_"
     export GLOBAL_ATTRS_URI="metadata/chumash/globalAttributes.yaml"
     export VARIABLE_ATTRS_URI="metadata/chumash/variableAttributes.yaml"
-    export MAX_SEGMENTS={{max_segments}}
     export CLOUD_TMP_DIR="cloud_tmp_chumash"
     export REMOVE_DOWNLOADED_FILES=no
     export PYTHONPATH=.

src/__init__.py

@@ -5,7 +5,7 @@
 from src.misc_helper import error, info
 
 
-def save_dataset_to_netcdf(ds: xr.Dataset, filename: str):
+def save_dataset_to_netcdf(ds: xr.Dataset, filename: str) -> bool:
     info(f"  - saving dataset to: {filename}")
     try:
         ds.to_netcdf(
@@ -17,8 +17,10 @@ def save_dataset_to_netcdf(ds: xr.Dataset, filename: str):
                 "sensitivity": {"_FillValue": None},
             },
         )
+        return True
     except Exception as e:  # pylint: disable=broad-exception-caught
         error(f"Unable to save {filename}: {e}")
+        return False
 
 
 def save_dataset_to_csv(ds: xr.Dataset, filename: str):

src/process_helper.py

@@ -144,7 +144,6 @@ def process_day(self, date: str) -> Optional[List[str]]:
 
         result: Optional[ProcessResult] = self.pypam_support.process_captured_segments(
             sensitivity_da=self.sensitivity_da,
-            sensitivity_flat_value=self.sensitivity_flat_value,
         )
 
         if result is None:
@@ -246,6 +245,10 @@ def process_segment_at_hour_minute(self, at_hour: int, at_minute: int):
         if self.voltage_multiplier is not None:
             audio_segment *= self.voltage_multiplier
 
+        if self.sensitivity_flat_value is not None:
+            # convert signal to uPa
+            audio_segment = audio_segment * 10 ** (self.sensitivity_flat_value / 20)
+
         self.pypam_support.add_segment(dt, audio_segment)
 
     def _get_global_attributes(self, year: int, month: int, day: int):

src/pypam_support.py

@@ -136,7 +136,6 @@ def add_segment(self, dt: datetime, data: np.ndarray):
     def process_captured_segments(
         self,
         sensitivity_da: Optional[xr.DataArray] = None,
-        sensitivity_flat_value: Optional[float] = None,
     ) -> Optional[ProcessResult]:
         """
         Gets the resulting hybrid millidecade bands for all captured segments.
@@ -179,7 +178,6 @@ def process_captured_segments(
             times=times,
             spectra=spectra,
             sensitivity_da=sensitivity_da,
-            sensitivity_flat_value=sensitivity_flat_value,
         )
 
         effort_da = xr.DataArray(
@@ -194,7 +192,6 @@ def _get_aggregated_milli_psd(
         times: List[np.int64],
         spectra: List[np.ndarray],
         sensitivity_da: Optional[xr.DataArray] = None,
-        sensitivity_flat_value: Optional[float] = None,
     ) -> xr.DataArray:
         # Convert the spectra to a DataArray
         psd_da = xr.DataArray(
@@ -205,7 +202,7 @@ def _get_aggregated_milli_psd(
 
         psd_da = self._spectra_to_bands(psd_da)
         debug(f"  {psd_da.frequency_bins=}")
-        psd_da = _apply_sensitivity(psd_da, sensitivity_da, sensitivity_flat_value)
+        psd_da = _apply_sensitivity(psd_da, sensitivity_da)
 
         # just need single precision:
         psd_da = psd_da.astype(np.float32)
@@ -257,7 +254,6 @@ def _get_spectrum(data: np.ndarray, fs: int, nfft: int) -> Tuple[np.ndarray, np.
 def _apply_sensitivity(
     psd_da: xr.DataArray,
     sensitivity_da: Optional[xr.DataArray],
-    sensitivity_flat_value: Optional[float] = None,
 ) -> xr.DataArray:
     psd_da = cast(xr.DataArray, 10 * np.log10(psd_da))
 
@@ -266,14 +262,12 @@ def _apply_sensitivity(
     # (previously, subtraction)
     # 2023-06-12: Back to subtraction (as we're focusing on MARS data at the moment)
     # TODO but this is still one pending aspect to finalize.
+    # 2023-08-03: sensitivity_flat_value is handled upstream now.
 
     if sensitivity_da is not None:
         freq_subset = sensitivity_da.interp(frequency=psd_da.frequency_bins)
         info(f"  Applying sensitivity({len(freq_subset.values)})={freq_subset}")
         psd_da -= freq_subset.values
-    elif sensitivity_flat_value is not None:
-        info(f"  Applying {sensitivity_flat_value=}")
-        psd_da -= sensitivity_flat_value
 
     return psd_da