From eb8db7ff5dddbe5057f5235cab7d0e18c43d8f2d Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Sun, 23 Jun 2024 15:55:56 +0200 Subject: [PATCH 01/36] add arg to load only wfs and do not extract charges --- .../ggrolleron/load_wfs_compute_charge.py | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py index 7da5dd92..c82e6393 100644 --- a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py +++ b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py @@ -39,6 +39,12 @@ type=int, ) +parser.add_argument( + "--only_wfs", + action="store_true", + default=False, + help="to only reload wfs", +) # boolean arguments parser.add_argument( "--reload_wfs", @@ -124,7 +130,7 @@ def main( for _run_number, _max_events in zip(run_number, max_events): try: - if kwargs.get("reload_wfs", False): + if kwargs.get("only_wfs", False) or kwargs.get("reload_wfs", False): log.info("reloading waveforms") tool = WaveformsNectarCAMCalibrationTool( progress_bar=True, @@ -136,16 +142,17 @@ def main( tool.start() tool.finish() - tool = ChargesNectarCAMCalibrationTool( - progress_bar=True, - run_number=_run_number, - max_events=_max_events, - from_computed_waveforms=True, - **charges_kwargs, - ) - tool.setup() - tool.start() - tool.finish() + if not (kwargs.get("only_wfs", False)): + tool = ChargesNectarCAMCalibrationTool( + progress_bar=True, + run_number=_run_number, + max_events=_max_events, + from_computed_waveforms=True, + **charges_kwargs, + ) + tool.setup() + tool.start() + tool.finish() else: log.info("trying to compute charges from waveforms yet extracted") tool = ChargesNectarCAMCalibrationTool( @@ -203,5 +210,8 @@ def main( kwargs.pop("verbosity") log.info(f"arguments passed to main are : {kwargs}") - + # kwargs['reload_wfs'] = True + # kwargs['run_number'] = [3938] + # kwargs['overwrite'] = True + # kwargs['events_per_slice'] = 2000 main(log=log, **kwargs) From 2b8b613f9b7be62ae33fdf563bc1f3fd8670ede1 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 26 Jun 2024 19:09:06 +0200 Subject: [PATCH 02/36] cleaner way to treat containers inside makers --- src/nectarchain/data/container/core.py | 38 +++++--- .../calibration/gain/FlatFieldSPEMakers.py | 45 ++++++---- .../calibration/gain/photostat_makers.py | 90 ++++++++++++------- src/nectarchain/makers/chargesMakers.py | 29 ++++-- 4 files changed, 131 insertions(+), 71 deletions(-) diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index 87e41c0d..4635a878 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -54,7 +54,7 @@ class NectarCAMContainer(Container): """ @staticmethod - def _container_from_hdf5(path, container_class): + def _container_from_hdf5(path, container_class, index_component=0): """ Static method to read a container from an HDF5 file. @@ -74,7 +74,7 @@ def _container_from_hdf5(path, container_class): container = container_class() with HDF5TableReader(path) as reader: tableReader = reader.read( - table_name=f"/data/{container_class.__name__}", + table_name=f"/data/{container_class.__name__}_{index_component}", containers=container_class, ) container = next(tableReader) @@ -82,7 +82,7 @@ def _container_from_hdf5(path, container_class): yield container @classmethod - def from_hdf5(cls, path): + def from_hdf5(cls, path, index_component=0): """ Reads a container from an HDF5 file. @@ -99,7 +99,9 @@ def from_hdf5(cls, path): >>> container = NectarCAMContainer.from_hdf5('path_to_file.h5') """ - return cls._container_from_hdf5(path, container_class=cls) + return cls._container_from_hdf5( + path, container_class=cls, index_component=index_component + ) class ArrayDataContainer(NectarCAMContainer): @@ -197,7 +199,7 @@ class TriggerMapContainer(Container): ) @classmethod - def from_hdf5(cls, path, slice_index=None): + def from_hdf5(cls, path, slice_index=None, index_component=0): """ Reads a container from an HDF5 file. @@ -215,11 +217,16 @@ def from_hdf5(cls, path, slice_index=None): """ return cls._container_from_hdf5( - path, slice_index=slice_index, container_class=cls + path, + slice_index=slice_index, + container_class=cls, + index_component=index_component, ) @staticmethod - def _container_from_hdf5(path, container_class, slice_index=None): + def _container_from_hdf5( + path, container_class, slice_index=None, index_component=0 + ): """ Reads a container from an HDF5 file. @@ -259,18 +266,20 @@ def _container_from_hdf5(path, container_class, slice_index=None): # container.containers[data] = eval(f"module.{container_class.__name__}s")() for key, trigger in EventType.__members__.items(): try: + _container = eval( + f"module.{container.fields['containers'].default_factory.args[0].__name__}" + ) waveforms_data = eval( f"reader._h5file.root.{data}.__members__" ) _mask = [ - container_class.__name__ in _word - for _word in waveforms_data + _container.__name__ in _word for _word in waveforms_data ] _waveforms_data = np.array(waveforms_data)[_mask] if len(_waveforms_data) == 1: tableReader = reader.read( table_name=f"/{data}/{_waveforms_data[0]}/{trigger.name}", - containers=container_class, + containers=_container, ) # container.containers[data].containers[trigger] = next(tableReader) container.containers[trigger] = next(tableReader) @@ -298,11 +307,12 @@ def _container_from_hdf5(path, container_class, slice_index=None): data = f"data_{slice_index}" for key, trigger in EventType.__members__.items(): try: + _container = eval( + f"module.{container.fields['containers'].default_factory.args[0].__name__}" + ) tableReader = reader.read( - table_name=f"/{data}/{trigger.name}", - containers=eval( - f"module.{container.fields['containers'].default_factory.args[0].__name__}" - ), + table_name=f"/{data}/{_container.__name__}_{index_component}/{trigger.name}", + containers=_container, ) container.containers[trigger] = next(tableReader) except NoSuchNodeError as err: diff --git a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py index 8f6948e9..516ec85c 100644 --- a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py @@ -108,25 +108,38 @@ def start( ) else: self.log.info(f"reading computed charge from files {files[0]}") - chargesContainers = ChargesContainer.from_hdf5(files[0]) + chargesContainers = ChargesContainers.from_hdf5(files[0]) if isinstance(chargesContainers, ChargesContainer): self.components[0]._chargesContainers = chargesContainers - elif isinstance(chargesContainers, ChargesContainers): - self.log.debug("merging along TriggerType") - self.components[0]._chargesContainers = merge_map_ArrayDataContainer( - chargesContainers - ) else: - self.log.debug("merging along slices") - chargesContaienrs_merdes_along_slices = ( - ArrayDataComponent.merge_along_slices( - containers_generator=chargesContainers + n_slices = 0 + try: + while True: + next(chargesContainers) + n_slices += 1 + except StopIteration: + pass + chargesContainers = ChargesContainers.from_hdf5(files[0]) + if n_slices == 1: + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._chargesContainers = merge_map_ArrayDataContainer( + chargesContainers + ) + else: + self.log.info("merging along slices") + chargesContaienrs_merdes_along_slices = ( + ArrayDataComponent.merge_along_slices( + containers_generator=chargesContainers + ) + ) + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._chargesContainers = merge_map_ArrayDataContainer( + chargesContaienrs_merdes_along_slices ) - ) - self.log.debug("merging along TriggerType") - self.components[0]._chargesContainers = merge_map_ArrayDataContainer( - chargesContaienrs_merdes_along_slices - ) def _write_container(self, container: Container, index_component: int = 0) -> None: # if isinstance(container,SPEfitContainer) : @@ -173,7 +186,7 @@ class FlatFieldSPENominalStdNectarCAMCalibrationTool( class FlatFieldSPECombinedStdNectarCAMCalibrationTool( FlatFieldSPENominalNectarCAMCalibrationTool ): - name = "FlatFieldCombinedStddNectarCAM" + name = "FlatFieldCombinedStdNectarCAM" componentsList = ComponentNameList( NectarCAMComponent, default_value=["FlatFieldCombinedSPEStdNectarCAMComponent"], diff --git a/src/nectarchain/makers/calibration/gain/photostat_makers.py b/src/nectarchain/makers/calibration/gain/photostat_makers.py index 302fac51..4a2b426d 100644 --- a/src/nectarchain/makers/calibration/gain/photostat_makers.py +++ b/src/nectarchain/makers/calibration/gain/photostat_makers.py @@ -115,46 +115,72 @@ def start( ) else: self.log.info(f"reading computed charge from FF file {FF_files[0]}") - chargesContainers = ChargesContainer.from_hdf5(FF_files[0]) + chargesContainers = ChargesContainers.from_hdf5(FF_files[0]) if isinstance(chargesContainers, ChargesContainer): self.components[0]._FF_chargesContainers = chargesContainers - elif isinstance(chargesContainers, ChargesContainers): - self.log.debug("merging along TriggerType") - self.components[0]._FF_chargesContainers = merge_map_ArrayDataContainer( - chargesContainers - ) else: - self.log.debug("merging along slices") - chargesContaienrs_merdes_along_slices = ( - ArrayDataComponent.merge_along_slices(chargesContainers) - ) - self.log.debug("merging along TriggerType") - self.components[0]._FF_chargesContainers = merge_map_ArrayDataContainer( - chargesContaienrs_merdes_along_slices - ) + n_slices = 0 + try: + while True: + next(chargesContainers) + n_slices += 1 + except StopIteration: + pass + chargesContainers = ChargesContainers.from_hdf5(FF_files[0]) + if n_slices == 1: + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._FF_chargesContainers = merge_map_ArrayDataContainer( + chargesContainers + ) + else: + self.log.info("merging along slices") + chargesContaienrs_merdes_along_slices = ( + ArrayDataComponent.merge_along_slices( + containers_generator=chargesContainers + ) + ) + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._FF_chargesContainers = merge_map_ArrayDataContainer( + chargesContaienrs_merdes_along_slices + ) self.log.info(f"reading computed charge from Ped file {Ped_files[0]}") - chargesContainers = ChargesContainer.from_hdf5(Ped_files[0]) + chargesContainers = ChargesContainers.from_hdf5(Ped_files[0]) if isinstance(chargesContainers, ChargesContainer): self.components[0]._Ped_chargesContainers = chargesContainers - elif isinstance(chargesContainers, ChargesContainers): - self.log.debug("merging along TriggerType") - self.components[ - 0 - ]._Ped_chargesContainers = merge_map_ArrayDataContainer( - chargesContainers - ) else: - self.log.debug("merging along slices") - chargesContaienrs_merdes_along_slices = ( - ArrayDataComponent.merge_along_slices(chargesContainers) - ) - self.log.debug("merging along TriggerType") - self.components[ - 0 - ]._Ped_chargesContainers = merge_map_ArrayDataContainer( - chargesContaienrs_merdes_along_slices - ) + n_slices = 0 + try: + while True: + next(chargesContainers) + n_slices += 1 + except StopIteration: + pass + chargesContainers = ChargesContainers.from_hdf5(Ped_files[0]) + if n_slices == 1: + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._Ped_chargesContainers = merge_map_ArrayDataContainer( + chargesContainers + ) + else: + self.log.info("merging along slices") + chargesContaienrs_merdes_along_slices = ( + ArrayDataComponent.merge_along_slices( + containers_generator=chargesContainers + ) + ) + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._Ped_chargesContainers = merge_map_ArrayDataContainer( + chargesContaienrs_merdes_along_slices + ) def _write_container(self, container: Container, index_component: int = 0) -> None: # if isinstance(container,SPEfitContainer) : diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/chargesMakers.py index 97ff9cb2..7fad5e43 100644 --- a/src/nectarchain/makers/chargesMakers.py +++ b/src/nectarchain/makers/chargesMakers.py @@ -8,6 +8,7 @@ import pathlib import numpy as np +from ctapipe.containers import EventType from ctapipe.core.traits import Bool, ComponentNameList from ctapipe.image.extractor import ( BaselineSubtractedNeighborPeakWindowSum, @@ -20,7 +21,12 @@ TwoPassWindowSum, ) -from ..data.container import ChargesContainers, WaveformsContainer, WaveformsContainers +from ..data.container import ( + ChargesContainers, + TriggerMapContainer, + WaveformsContainer, + WaveformsContainers, +) from ..data.management import DataManagement from .component import ChargesComponent, NectarCAMComponent from .core import EventsLoopNectarCAMCalibrationTool @@ -69,6 +75,7 @@ def start( *args, **kwargs, ): + ##cette implémentation est complétement nulle if self.from_computed_waveforms: files = DataManagement.find_waveforms( run_number=self.run_number, max_events=self.max_events @@ -87,17 +94,21 @@ def start( self.log.info( f"{files[0]} is the computed wavforms files found with max_events >= {self.max_events} for run {self.run_number}" ) - waveformsContainers = WaveformsContainer.from_hdf5(files[0]) + waveformsContainers = WaveformsContainers.from_hdf5(files[0]) if not (isinstance(waveformsContainers, WaveformsContainer)): - chargesContainers = ChargesContainers() - if isinstance(waveformsContainers, WaveformsContainers): - self.log.debug( - "WaveformsContainer file container multiple trigger type" - ) + n_slices = 0 + try: + while True: + next(waveformsContainers) + n_slices += 1 + except StopIteration: + pass + waveformsContainers = WaveformsContainers.from_hdf5(files[0]) + if n_slices == 1: self._init_writer(sliced=False) chargesContainers = ( ChargesComponent._create_from_waveforms_looping_eventType( - waveformsContainers=waveformsContainers, + waveformsContainers=next(waveformsContainers), subarray=self.event_source.subarray, method=self.method, **self.extractor_kwargs, @@ -106,7 +117,7 @@ def start( self._write_container(container=chargesContainers) else: self.log.debug( - "WaveformsContainer file container multiple slices of the run events" + f"WaveformsContainer file contains {n_slices} slices of the run events" ) for slice_index, _waveformsContainers in enumerate( waveformsContainers From 5f308513a1f49e67efa2921f43f9e84942747c5d Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 26 Jun 2024 19:10:39 +0200 Subject: [PATCH 03/36] new methods to find computed calibration quatities ds to find computed calibration quatities --- src/nectarchain/data/management.py | 35 ++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index a8ebf5d5..8bf0dae5 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -261,14 +261,45 @@ def find_charges( data_type="charges", ) - def find_SPE_HHV(run_number, method="FullWaveformSum", str_extractor_kwargs=""): + def find_SPE_combined( + run_number, method="FullWaveformSum", str_extractor_kwargs="" + ): + return __class__.find_SPE_HHV( + run_number=run_number, + method=method, + str_extractor_kwargs=str_extractor_kwargs, + keyword="FlatFieldCombined", + ) + + def find_SPE_nominal( + run_number, method="FullWaveformSum", str_extractor_kwargs="", free_pp_n=False + ): + return __class__.find_SPE_HHV( + run_number=run_number, + method=method, + str_extractor_kwargs=str_extractor_kwargs, + free_pp_n=free_pp_n, + keyword="FlatFieldSPENominal", + ) + + def find_SPE_HHV( + run_number, + method="FullWaveformSum", + str_extractor_kwargs="", + free_pp_n=False, + **kwargs, + ): + keyword = kwargs.get("keyword", "FlatFieldSPEHHV") + std_key = "" if free_pp_n else "Std" full_file = glob.glob( pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/SPEfit/" - f"FlatFieldSPEHHVStdNectarCAM_run{run_number}_{method}" + f"{keyword}{std_key}NectarCAM_run{run_number}*_{method}" f"_{str_extractor_kwargs}.h5" ).__str__() ) + ###need to improve the files search !! + # -> unstable behavior with SPE results computed with maxevents not to None if len(full_file) != 1: all_files = glob.glob( pathlib.Path( From d8aa001c0f051c54714c25e431f49c7fbf16ea51 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 26 Jun 2024 19:11:48 +0200 Subject: [PATCH 04/36] update to support new version of EvB v6 --- src/nectarchain/makers/component/core.py | 4 ++-- src/nectarchain/makers/component/waveformsComponent.py | 3 ++- src/nectarchain/makers/core.py | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/nectarchain/makers/component/core.py b/src/nectarchain/makers/component/core.py index ee5b1470..3c348f2e 100644 --- a/src/nectarchain/makers/component/core.py +++ b/src/nectarchain/makers/component/core.py @@ -54,7 +54,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - self.__pixels_id = parent._event_source.camera_config.expected_pixels_id + self.__pixels_id = parent._event_source.nectarcam_service.pixel_ids self.__run_number = parent.run_number self.__npixels = parent.npixels @@ -111,7 +111,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs ) - self.__nsamples = parent._event_source.camera_config.num_samples + self.__nsamples = parent._event_source.nectarcam_service.num_samples self.trigger_list = [] diff --git a/src/nectarchain/makers/component/waveformsComponent.py b/src/nectarchain/makers/component/waveformsComponent.py index 74e13237..50460408 100644 --- a/src/nectarchain/makers/component/waveformsComponent.py +++ b/src/nectarchain/makers/component/waveformsComponent.py @@ -230,7 +230,8 @@ def sort(waveformsContainer: WaveformsContainer, method: str = "event_id"): @staticmethod def select_waveforms_hg( - waveformsContainer: WaveformsContainer, pixel_id: np.ndarray + waveformsContainer: WaveformsContainer, + pixel_id: np.ndarray, ): """Select HIGH GAIN waveforms from the container. diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 08a18fe3..3ed6a6cb 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -217,7 +217,7 @@ def _get_provided_component_kwargs(self, componentName: str): output_component_kwargs[key] = getattr(self, key) return output_component_kwargs - def _init_writer(self, sliced: bool = False, slice_index: int = 0, group_name = None): + def _init_writer(self, sliced: bool = False, slice_index: int = 0, group_name=None): if hasattr(self, "writer"): self.writer.close() @@ -310,7 +310,7 @@ def setup(self, *args, **kwargs): def _setup_eventsource(self, *args, **kwargs): self._load_eventsource(*args, **kwargs) self.__npixels = self._event_source.camera_config.num_pixels - self.__pixels_id = self._event_source.camera_config.expected_pixels_id + self.__pixels_id = self._event_source.nectarcam_service.pixel_ids def _setup_components(self, *args, **kwargs): self.log.info("setup of components") @@ -422,7 +422,7 @@ def _write_container(self, container: Container, index_component: int = 0) -> No container.validate() if isinstance(container, NectarCAMContainer): self.writer.write( - table_name=str(container.__class__.__name__), + table_name=f"{container.__class__.__name__}_{index_component}", containers=container, ) elif isinstance(container, TriggerMapContainer): From d183536aba96bc9d1545e791af0d85c205a599ea Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 26 Jun 2024 19:15:39 +0200 Subject: [PATCH 05/36] -bug fix SPE combined : method use to update parameters was not the wanted one -update pp and n values to last studies -display wfs -> mean instead of sum -Agg backend is not forced in Pstat calibnration --- src/nectarchain/display/display.py | 6 ++-- .../component/photostatistic_algorithm.py | 2 +- .../parameters_SPECombined_fromHHVFit.yaml | 4 +-- .../component/spe/parameters_SPEHHV.yaml | 4 +-- .../component/spe/parameters_SPEHHVStd.yaml | 4 +-- .../component/spe/parameters_SPEnominal.yaml | 4 +-- .../spe/parameters_SPEnominalStd.yaml | 4 +-- .../spe/parameters_signal_combined.yaml | 4 +-- .../makers/component/spe/spe_algorithm.py | 29 ++++++++++++------- 9 files changed, 34 insertions(+), 27 deletions(-) diff --git a/src/nectarchain/display/display.py b/src/nectarchain/display/display.py index 9df8a759..73d57469 100644 --- a/src/nectarchain/display/display.py +++ b/src/nectarchain/display/display.py @@ -29,7 +29,7 @@ def display(container: ArrayDataContainer, evt, geometry, cmap="gnuplot2"): image = container.charges_hg pixels_id = container.pixels_id elif isinstance(container, WaveformsContainer): - image = container.wfs_hg.sum(axis=2) + image = container.wfs_hg.mean(axis=2) pixels_id = container.pixels_id else: log.error( @@ -55,8 +55,8 @@ def display(container: ArrayDataContainer, evt, geometry, cmap="gnuplot2"): disp = CameraDisplay(geometry=geometry, image=image[evt], cmap=cmap) disp.highlight_pixels(highlighten_pixels, color="r", linewidth=2) - disp.add_colorbar() - return disp + disp.add_colorbar(label="ADC") + return {"disp": disp, "highlighten_pixels": highlighten_pixels} @staticmethod def plot_waveform(waveformsContainer: WaveformsContainer, evt, **kwargs): diff --git a/src/nectarchain/makers/component/photostatistic_algorithm.py b/src/nectarchain/makers/component/photostatistic_algorithm.py index 2b622aa1..21ba8e26 100644 --- a/src/nectarchain/makers/component/photostatistic_algorithm.py +++ b/src/nectarchain/makers/component/photostatistic_algorithm.py @@ -169,7 +169,7 @@ def plot_correlation( Returns: fig (plt.Figure): The figure object containing the scatter plot and the linear fit line. """ - matplotlib.use("TkAgg") + # matplotlib.use("TkAgg") # Create a mask to filter the data points based on certain criteria mask = (photoStat_gain > 20) * (SPE_gain > 0) * (photoStat_gain < 80) diff --git a/src/nectarchain/makers/component/spe/parameters_SPECombined_fromHHVFit.yaml b/src/nectarchain/makers/component/spe/parameters_SPECombined_fromHHVFit.yaml index 62515304..aeb19cd6 100644 --- a/src/nectarchain/makers/component/spe/parameters_SPECombined_fromHHVFit.yaml +++ b/src/nectarchain/makers/component/spe/parameters_SPECombined_fromHHVFit.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : .NAN, max : .NAN }, @@ -20,7 +20,7 @@ max : .NAN }, n : { - value : 0.697, + value : 0.713, min : .NAN, max : .NAN }, diff --git a/src/nectarchain/makers/component/spe/parameters_SPEHHV.yaml b/src/nectarchain/makers/component/spe/parameters_SPEHHV.yaml index 89ece153..5ece2735 100644 --- a/src/nectarchain/makers/component/spe/parameters_SPEHHV.yaml +++ b/src/nectarchain/makers/component/spe/parameters_SPEHHV.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : 0.2, max : 0.8 }, @@ -20,7 +20,7 @@ max : 0.7 }, n : { - value : 0.697, + value : 0.713, min : 0.5, max : 0.9 }, diff --git a/src/nectarchain/makers/component/spe/parameters_SPEHHVStd.yaml b/src/nectarchain/makers/component/spe/parameters_SPEHHVStd.yaml index 12746d7f..c9989f1f 100644 --- a/src/nectarchain/makers/component/spe/parameters_SPEHHVStd.yaml +++ b/src/nectarchain/makers/component/spe/parameters_SPEHHVStd.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : .NAN, max : .NAN }, @@ -20,7 +20,7 @@ max : 0.7 }, n : { - value : 0.697, + value : 0.713, min : .NAN, max : .NAN }, diff --git a/src/nectarchain/makers/component/spe/parameters_SPEnominal.yaml b/src/nectarchain/makers/component/spe/parameters_SPEnominal.yaml index 18b6ee31..914a8b15 100644 --- a/src/nectarchain/makers/component/spe/parameters_SPEnominal.yaml +++ b/src/nectarchain/makers/component/spe/parameters_SPEnominal.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : 0.2, max : 0.8 }, @@ -20,7 +20,7 @@ max : 0.7 }, n : { - value : 0.697, + value : 0.713, min : 0.5, max : 0.9 }, diff --git a/src/nectarchain/makers/component/spe/parameters_SPEnominalStd.yaml b/src/nectarchain/makers/component/spe/parameters_SPEnominalStd.yaml index 3be2f77f..6c87ecae 100644 --- a/src/nectarchain/makers/component/spe/parameters_SPEnominalStd.yaml +++ b/src/nectarchain/makers/component/spe/parameters_SPEnominalStd.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : .NAN, max : .NAN }, @@ -20,7 +20,7 @@ max : 0.7 }, n : { - value : 0.697, + value : 0.713, min : .NAN, max : .NAN }, diff --git a/src/nectarchain/makers/component/spe/parameters_signal_combined.yaml b/src/nectarchain/makers/component/spe/parameters_signal_combined.yaml index a04fc05a..080bb852 100644 --- a/src/nectarchain/makers/component/spe/parameters_signal_combined.yaml +++ b/src/nectarchain/makers/component/spe/parameters_signal_combined.yaml @@ -15,7 +15,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : .NAN, max : .NAN }, @@ -25,7 +25,7 @@ max : 0.7 }, n : { - value : 0.697, + value : 0.713, min : .NAN, max : .NAN }, diff --git a/src/nectarchain/makers/component/spe/spe_algorithm.py b/src/nectarchain/makers/component/spe/spe_algorithm.py index c1c6607e..b9f280a7 100644 --- a/src/nectarchain/makers/component/spe/spe_algorithm.py +++ b/src/nectarchain/makers/component/spe/spe_algorithm.py @@ -641,7 +641,7 @@ def _make_minuitParameters_array_from_parameters( for i, _id in enumerate(pixels_id): index = np.where(self.pixels_id == _id)[0][0] - parameters = __class__._update_parameters( + parameters = self._update_parameters( self.parameters, self._charge[index].data[~self._charge[index].mask], self._counts[index].data[~self._charge[index].mask], @@ -893,6 +893,7 @@ def plot_single_matplotlib( pedestalWidth: float, luminosity: float, likelihood: float, + **kwargs, ) -> tuple: """ Generate a plot of the data and a model fit for a specific pixel. @@ -914,7 +915,11 @@ def plot_single_matplotlib( Returns: tuple: A tuple containing the generated plot figure and the axes of the plot. """ - fig, ax = plt.subplots(1, 1, figsize=(8, 8)) + if kwargs.get("ax", False) and kwargs.get("fig", False): + fig = kwargs.get("fig") + ax = kwargs.get("ax") + else: + fig, ax = plt.subplots(1, 1, figsize=(8, 8)) ax.errorbar(charge, counts, np.sqrt(counts), zorder=0, fmt=".", label="data") ax.plot( charge, @@ -1018,7 +1023,7 @@ class SPEHHValgorithm(SPEnominalalgorithm): help="The name of the SPE fit parameters file", ).tag(config=True) tol = Float( - 1e40, + 1e5, help="The tolerance used for minuit", read_only=True, ).tag(config=True) @@ -1079,7 +1084,7 @@ class SPEHHVStdalgorithm(SPEnominalStdalgorithm): help="The name of the SPE fit parameters file", ).tag(config=True) tol = Float( - 1e40, + 1e5, help="The tolerance used for minuit", read_only=True, ).tag(config=True) @@ -1093,7 +1098,7 @@ class SPECombinedalgorithm(SPEnominalalgorithm): ).tag(config=True) tol = Float( - 1e5, + 1e-1, help="The tolerance used for minuit", read_only=True, ).tag(config=True) @@ -1135,7 +1140,7 @@ def __init__( ) self.__fix_parameters() - self._nectarGainSPEresult = SPEfitContainer.from_hdf5(self.SPE_result) + self._nectarGainSPEresult = next(SPEfitContainer.from_hdf5(self.SPE_result)) if ( len( pixels_id[ @@ -1172,7 +1177,9 @@ def __fix_parameters(self) -> None: luminosity = self._parameters["luminosity"] luminosity.frozen = True - def _make_fit_array_from_parameters(self, pixels_id=None, **kwargs): + def _make_minuitParameters_array_from_parameters( + self, pixels_id: np.ndarray = None, **kwargs + ) -> np.ndarray: """ Generates the fit array from the fixed parameters and the fitted data obtained from a 1400V run. @@ -1183,7 +1190,7 @@ def _make_fit_array_from_parameters(self, pixels_id=None, **kwargs): Returns: array-like: The fit array. """ - return super()._make_fit_array_from_parameters( + return super()._make_minuitParameters_array_from_parameters( pixels_id=pixels_id, nectarGainSPEresult=self._nectarGainSPEresult, **kwargs, @@ -1222,9 +1229,9 @@ def _update_parameters( index = np.where(pixel_id == nectarGainSPEresult.pixels_id)[0][0] - resolution.value = nectarGainSPEresult.resolution[index] - pp.value = nectarGainSPEresult.pp[index] - n.value = nectarGainSPEresult.n[index]["n"] + resolution.value = nectarGainSPEresult.resolution[index][0] + pp.value = nectarGainSPEresult.pp[index][0] + n.value = nectarGainSPEresult.n[index][0] if luminosity.frozen: luminosity.value = nectarGainSPEresult.luminosity[index].value From b46a959c07432de79a7d33fad9a725b8b76a240a Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 9 Dec 2024 14:58:00 +0100 Subject: [PATCH 06/36] add find photostat result method --- src/nectarchain/data/management.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index 8bf0dae5..cc3599e9 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -260,6 +260,27 @@ def find_charges( ext=f"_{method}_{str_extractor_kwargs}.h5", data_type="charges", ) + + def find_photostat( + FF_run_number, + ped_run_number, + FF_method="FullWaveformSum", + ped_method="FullWaveformSum", + str_extractor_kwargs="", + ): + full_file = glob.glob( + pathlib.Path( + f"{os.environ.get('NECTARCAMDATA','/tmp')}/PhotoStat/" + f"PhotoStatisticNectarCAM_FFrun{FF_run_number}_{FF_method}" + f"_{str_extractor_kwargs}_Pedrun{ped_run_number}_{ped_method}.h5" + ).__str__() + ) + log.debug('for now it does not check if there are files with max events') + if len(full_file) != 1 : + raise Exception(f"the files is {full_file}") + return full_file + + def find_SPE_combined( run_number, method="FullWaveformSum", str_extractor_kwargs="" From 75f6b345a1229fecad1c478b19fb643d08aed4a1 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 9 Dec 2024 14:59:20 +0100 Subject: [PATCH 07/36] user scripts --- src/nectarchain/data/management.py | 18 ++++++------- .../ggrolleron/gain_PhotoStat_computation.py | 25 +++++++++++++++---- .../gain_SPEfit_combined_computation.py | 16 +++++++++--- .../ggrolleron/gain_SPEfit_computation.py | 10 +++++++- .../ggrolleron/load_wfs_compute_charge.py | 8 +++--- .../ggrolleron/load_wfs_compute_charge.sh | 3 ++- .../script_loop_charge_extraction.sh | 20 +++++++++++++++ .../ggrolleron/script_loop_gain_HHV.sh | 20 +++++++++++++++ .../ggrolleron/script_loop_gain_HHV_free.sh | 20 +++++++++++++++ .../ggrolleron/script_loop_gain_nominal.sh | 20 +++++++++++++++ .../script_loop_gain_nominal_from_HHV.sh | 21 ++++++++++++++++ .../ggrolleron/script_loop_gain_pstat.sh | 14 +++++++++++ 12 files changed, 170 insertions(+), 25 deletions(-) create mode 100755 src/nectarchain/user_scripts/ggrolleron/script_loop_charge_extraction.sh create mode 100755 src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV.sh create mode 100755 src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV_free.sh create mode 100755 src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal.sh create mode 100755 src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal_from_HHV.sh create mode 100755 src/nectarchain/user_scripts/ggrolleron/script_loop_gain_pstat.sh diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index cc3599e9..3a003c01 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -260,13 +260,13 @@ def find_charges( ext=f"_{method}_{str_extractor_kwargs}.h5", data_type="charges", ) - + def find_photostat( - FF_run_number, - ped_run_number, - FF_method="FullWaveformSum", - ped_method="FullWaveformSum", - str_extractor_kwargs="", + FF_run_number, + ped_run_number, + FF_method="FullWaveformSum", + ped_method="FullWaveformSum", + str_extractor_kwargs="", ): full_file = glob.glob( pathlib.Path( @@ -275,13 +275,11 @@ def find_photostat( f"_{str_extractor_kwargs}_Pedrun{ped_run_number}_{ped_method}.h5" ).__str__() ) - log.debug('for now it does not check if there are files with max events') - if len(full_file) != 1 : + log.debug("for now it does not check if there are files with max events") + if len(full_file) != 1: raise Exception(f"the files is {full_file}") return full_file - - def find_SPE_combined( run_number, method="FullWaveformSum", str_extractor_kwargs="" ): diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py index f3991be0..e7877ecc 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py @@ -60,13 +60,13 @@ "SlidingWindowMaxSum", "TwoPassWindowSum", ], - default="LocalPeakWindowSum", + default="GlobalPeakWindowSum", help="charge extractor method", type=str, ) parser.add_argument( "--extractor_kwargs", - default={"window_width": 16, "window_shift": 4}, + default={"window_width": 8, "window_shift": 4}, help="charge extractor kwargs", type=json.loads, ) @@ -124,10 +124,20 @@ def main( str_extractor_kwargs = CtapipeExtractor.get_extractor_kwargs_str( args.extractor_kwargs ) - path = DataManagement.find_SPE_HHV( + #path = DataManagement.find_SPE_HHV( + # run_number=args.HHV_run_number, + # method=args.method, + # str_extractor_kwargs=str_extractor_kwargs, + #) + #path = DataManagement.find_SPE_nominal( + # run_number=args.HHV_run_number, + # method=args.method, + # str_extractor_kwargs=str_extractor_kwargs, + #) + path = DataManagement.find_SPE_nominal( run_number=args.HHV_run_number, - method=args.method, - str_extractor_kwargs=str_extractor_kwargs, + method="GlobalPeakWindowSum", + str_extractor_kwargs=f"window_width_{8}_window_shift_4", ) if len(path) == 1: log.info( @@ -196,5 +206,10 @@ def main( kwargs.pop("figpath") kwargs.pop("HHV_run_number") + kwargs['FF_run_number'] = [3937] + kwargs['Ped_run_number'] = [3938] + kwargs['overwrite'] = True + args.HHV_run_number = 3936 + log.info(f"arguments passed to main are : {kwargs}") main(log=log, **kwargs) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py index 6cf83469..ef09117f 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py @@ -1,12 +1,12 @@ +import argparse +import copy +import glob import json import logging import os import sys import time from pathlib import Path -import argparse -import copy -import glob from nectarchain.data.management import DataManagement from nectarchain.makers.calibration import ( @@ -71,7 +71,7 @@ ) parser.add_argument( "--extractor_kwargs", - default={"window_width": 16, "window_shift": 4}, + default={"window_width": 10, "window_shift": 4}, help="charge extractor kwargs", type=json.loads, ) @@ -218,6 +218,14 @@ def main( kwargs.pop("display") kwargs.pop("HHV_run_number") + #args.HHV_run_number = 3942 + #kwargs['run_number'] = [3936] + #kwargs['overwrite'] = True + #kwargs['asked_pixels_id'] = [45,600,800] + #kwargs['multiproc'] = False + #args.display = True + #args.figpath = "/home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + log.info(f"arguments passed to main are : {kwargs}") main(log=log, **kwargs) log.info(f"time for execution is {time.time() - t:.2e} sec") diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py index a24a5ce3..2ea069e6 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py @@ -72,7 +72,7 @@ ) parser.add_argument( "--extractor_kwargs", - default={"window_width": 16, "window_shift": 4}, + default={"window_width": 8, "window_shift": 4}, help="charge extractor kwargs", type=json.loads, ) @@ -216,6 +216,14 @@ def main( kwargs.pop("HHV") kwargs.pop("free_pp_n") + #kwargs['run_number'] = [3942] + #kwargs['overwrite'] = True + #kwargs['asked_pixels_id'] = [45,600,800] + #args.HHV = True + #kwargs['multiproc'] = True + #args.display = True + #args.figpath = "/home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + log.info(f"arguments passed to main are : {kwargs}") main(log=log, **kwargs) log.info(f"time for execution is {time.time() - t:.2e} sec") diff --git a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py index c82e6393..83c901db 100644 --- a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py +++ b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py @@ -210,8 +210,8 @@ def main( kwargs.pop("verbosity") log.info(f"arguments passed to main are : {kwargs}") - # kwargs['reload_wfs'] = True - # kwargs['run_number'] = [3938] - # kwargs['overwrite'] = True - # kwargs['events_per_slice'] = 2000 + #kwargs['reload_wfs'] = True + #kwargs['run_number'] = [3784]#[5436] + #kwargs['overwrite'] = True + #kwargs['events_per_slice'] = 2000 main(log=log, **kwargs) diff --git a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.sh b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.sh index 0f050238..31150040 100644 --- a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.sh +++ b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.sh @@ -16,4 +16,5 @@ python load_wfs_compute_charge.py -s 2633 2634 3784 -f 2608 -p 2609 --spe_nevents 49227 49148 -1 --method LocalPeakWindowSum --extractor_kwargs '{"window_width":16,"window_shift":4}' -python load_wfs_compute_charge.py -r 3942 --max_events 100 --method GlobalPeakWindowSum --extractor_kwargs '{"window_width":16,"window_shift":4}' --overwrite -v debug \ No newline at end of file +python load_wfs_compute_charge.py -r 3942 --max_events 100 --method GlobalPeakWindowSum --extractor_kwargs '{"window_width":16,"window_shift":4}' --overwrite -v debug + diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_charge_extraction.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_charge_extraction.sh new file mode 100755 index 00000000..9c7ea100 --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_charge_extraction.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# 3936 3937 3938 3942 +#"FullWaveformSum" "LocalPeakWindowSum" "GlobalPeakWindowSum" +for METHOD in "FullWaveformSum" "LocalPeakWindowSum" "GlobalPeakWindowSum" +do + if [ $METHOD = "FullWaveformSum" ]; + then + export cmd="python load_wfs_compute_charge.py -r 3938 --events_per_slice 2000 --method $METHOD --overwrite -v INFO --reload_wfs" + echo $cmd + eval $cmd + else + for WIDTH in 8 9 10 11 12 16 + do + export cmd="python load_wfs_compute_charge.py -r 3938 --events_per_slice 2000 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO" + echo $cmd + eval $cmd + done + fi + #python load_wfs_compute_charge.py -r 3936 3942 --method LocalPeakWindowSum --extractor_kwargs "{'window_width':$WIDTH,'window_shift':4}" --overwrite -v INFO +done diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV.sh new file mode 100755 index 00000000..89343444 --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV.sh @@ -0,0 +1,20 @@ +#!/bin/bash +export _QT_QPA_PLATFORM=$QT_QPA_PLATFORM +export QT_QPA_PLATFORM=offscreen +for METHOD in "GlobalPeakWindowSum" +do + for WIDTH in 8 9 10 11 12 16 + do + if [ $WIDTH -eq 8 ]; then + export cmd="python gain_SPEfit_computation.py -r 3942 --HHV --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO --display --figpath /home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + echo $cmd + eval $cmd + else + export cmd="python gain_SPEfit_computation.py -r 3942 --HHV --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO" + echo $cmd + eval $cmd + fi + done +done +export QT_QPA_PLATFORM=$_QT_QPA_PLATFORM +unset _QT_QPA_PLATFORM diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV_free.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV_free.sh new file mode 100755 index 00000000..0029c258 --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV_free.sh @@ -0,0 +1,20 @@ +#!/bin/bash +export _QT_QPA_PLATFORM=$QT_QPA_PLATFORM +export QT_QPA_PLATFORM=offscreen +for METHOD in "GlobalPeakWindowSum" #"LocalPeakWindowSum" +do + for WIDTH in 8 10 12 + do + if [ $WIDTH -eq 8 ]; then + export cmd="python gain_SPEfit_computation.py -r 3942 --HHV --free_pp_n --multiproc --nproc 8 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO --display --figpath /home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + echo $cmd + eval $cmd + else + export cmd="python gain_SPEfit_computation.py -r 3942 --HHV --free_pp_n --multiproc --nproc 8 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO" + echo $cmd + eval $cmd + fi + done +done +export QT_QPA_PLATFORM=$_QT_QPA_PLATFORM +unset _QT_QPA_PLATFORM \ No newline at end of file diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal.sh new file mode 100755 index 00000000..60ed23de --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal.sh @@ -0,0 +1,20 @@ +#!/bin/bash +export _QT_QPA_PLATFORM=$QT_QPA_PLATFORM +export QT_QPA_PLATFORM=offscreen +for METHOD in "LocalPeakWindowSum" "GlobalPeakWindowSum" +do + for WIDTH in 8 9 10 12 + do + if [ $WIDTH -eq 8 ]; then + export cmd="python gain_SPEfit_computation.py -r 3936 --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO --display --figpath /home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + echo $cmd + eval $cmd + else + export cmd="python gain_SPEfit_computation.py -r 3936 --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO" + echo $cmd + eval $cmd + fi + done +done +export QT_QPA_PLATFORM=$_QT_QPA_PLATFORM +unset _QT_QPA_PLATFORM diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal_from_HHV.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal_from_HHV.sh new file mode 100755 index 00000000..f153d8f9 --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal_from_HHV.sh @@ -0,0 +1,21 @@ +#!/bin/bash +export _QT_QPA_PLATFORM=$QT_QPA_PLATFORM +export QT_QPA_PLATFORM=offscreen +for METHOD in "LocalPeakWindowSum" "GlobalPeakWindowSum" +do + for WIDTH in 9 11 16 + do + if [ $WIDTH -eq 8 ]; then + export cmd="python gain_SPEfit_combined_computation.py -r 3936 --HHV_run_number 3942 --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO --display --figpath /home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + echo $cmd + eval $cmd + else + export cmd="python gain_SPEfit_combined_computation.py -r 3936 --HHV_run_number 3942 --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO" + echo $cmd + eval $cmd + + fi + done +done +export QT_QPA_PLATFORM=$_QT_QPA_PLATFORM +unset _QT_QPA_PLATFORM \ No newline at end of file diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_pstat.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_pstat.sh new file mode 100755 index 00000000..e6d23cf6 --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_pstat.sh @@ -0,0 +1,14 @@ +#!/bin/bash +export _QT_QPA_PLATFORM=$QT_QPA_PLATFORM +export QT_QPA_PLATFORM=offscreen +for METHOD in "LocalPeakWindowSum" "GlobalPeakWindowSum" +do + for WIDTH in 8 9 10 11 12 16 + do + export cmd="python gain_PhotoStat_computation.py --FF_run_number 3937 --Ped_run_number 3938 --HHV_run_number 3936 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO --figpath /home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + echo $cmd + eval $cmd + done +done +export QT_QPA_PLATFORM=$_QT_QPA_PLATFORM +unset _QT_QPA_PLATFORM From 7452e6001918969f2262367d2c48c39a361eedbe Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 6 Jan 2025 15:13:11 +0100 Subject: [PATCH 08/36] add workflow_display to CI --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21e0d6a7..13f2b422 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: CI # by not building all branches on push, we avoid the duplicated builds in PRs on: + workflow_dispatch: push: # TODO # Remove comment: just a test to force CI on push From 5f9880045b284f47385476e1c5da5f6edcb7253b Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 6 Jan 2025 16:51:18 +0100 Subject: [PATCH 09/36] change tests name in user_scripts which is not a test for pytest --- .../user_scripts/dmousadi/{test_scripts => TRR_scripts}/README.md | 0 .../user_scripts/dmousadi/{test_scripts => TRR_scripts}/gui.py | 0 .../dmousadi/{test_scripts/tests => TRR_scripts/src}/__init__.py | 0 .../tests/deadtime_test.py => TRR_scripts/src/deadtime.py} | 0 .../tests/linearity_test.py => TRR_scripts/src/linearity.py} | 0 .../tests/pedestal_test.py => TRR_scripts/src/pedestal.py} | 0 .../src/pix_couple_tim_uncertainty.py} | 0 .../src/pix_tim_uncertainty.py} | 0 .../src/tools_components.py} | 0 .../trigger_timing_test.py => TRR_scripts/src/trigger_timing.py} | 0 .../dmousadi/{test_scripts/tests => TRR_scripts/src}/utils.py | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename src/nectarchain/user_scripts/dmousadi/{test_scripts => TRR_scripts}/README.md (100%) rename src/nectarchain/user_scripts/dmousadi/{test_scripts => TRR_scripts}/gui.py (100%) rename src/nectarchain/user_scripts/dmousadi/{test_scripts/tests => TRR_scripts/src}/__init__.py (100%) rename src/nectarchain/user_scripts/dmousadi/{test_scripts/tests/deadtime_test.py => TRR_scripts/src/deadtime.py} (100%) rename src/nectarchain/user_scripts/dmousadi/{test_scripts/tests/linearity_test.py => TRR_scripts/src/linearity.py} (100%) rename src/nectarchain/user_scripts/dmousadi/{test_scripts/tests/pedestal_test.py => TRR_scripts/src/pedestal.py} (100%) rename src/nectarchain/user_scripts/dmousadi/{test_scripts/tests/pix_couple_tim_uncertainty_test.py => TRR_scripts/src/pix_couple_tim_uncertainty.py} (100%) rename src/nectarchain/user_scripts/dmousadi/{test_scripts/tests/pix_tim_uncertainty_test.py => TRR_scripts/src/pix_tim_uncertainty.py} (100%) rename src/nectarchain/user_scripts/dmousadi/{test_scripts/tests/test_tools_components.py => TRR_scripts/src/tools_components.py} (100%) rename src/nectarchain/user_scripts/dmousadi/{test_scripts/tests/trigger_timing_test.py => TRR_scripts/src/trigger_timing.py} (100%) rename src/nectarchain/user_scripts/dmousadi/{test_scripts/tests => TRR_scripts/src}/utils.py (100%) diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/README.md b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/README.md similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/README.md rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/README.md diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/gui.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/gui.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/gui.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/gui.py diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/__init__.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/__init__.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/__init__.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/__init__.py diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/deadtime_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/deadtime_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/linearity_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/linearity_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pedestal_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pedestal_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pix_couple_tim_uncertainty_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pix_couple_tim_uncertainty_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pix_tim_uncertainty_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pix_tim_uncertainty_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/test_tools_components.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/test_tools_components.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/trigger_timing_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/trigger_timing_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/utils.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/utils.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/utils.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/utils.py From 08b956c615898c4b16a217f839cfe213b5b029d4 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 6 Jan 2025 17:11:52 +0100 Subject: [PATCH 10/36] fix import errors in TRR exluse user_scripts to pytest --- pyproject.toml | 3 +++ .../user_scripts/dmousadi/TRR_scripts/src/deadtime.py | 5 +++-- .../user_scripts/dmousadi/TRR_scripts/src/linearity.py | 5 +++-- .../user_scripts/dmousadi/TRR_scripts/src/pedestal.py | 5 +++-- .../dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py | 3 ++- .../dmousadi/TRR_scripts/src/pix_tim_uncertainty.py | 5 +++-- .../dmousadi/TRR_scripts/src/tools_components.py | 3 ++- .../user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py | 5 +++-- 8 files changed, 22 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a6fe588d..7e98a0d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,9 @@ dev = [ "setuptools_scm", ] +[tool.pytest.ini_options] +addopts = "--ignore=src/nectarchain/user_scripts" + docs = [ "sphinx", "sphinx-autodoc-typehints", diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py index f554cd78..f475a588 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py @@ -10,8 +10,9 @@ import numpy as np from astropy import units as u from iminuit import Minuit -from test_tools_components import DeadtimeTestTool -from utils import ( + +from .tools_components import DeadtimeTestTool +from .utils import ( ExponentialFitter, deadtime_and_expo_fit, deadtime_labels, diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py index 7c6b8d52..32a0ef84 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py @@ -9,8 +9,9 @@ import matplotlib.pyplot as plt import numpy as np from lmfit.models import Model -from test_tools_components import LinearityTestTool -from utils import ( + +from .tools_components import LinearityTestTool +from .utils import ( adc_to_pe, err_ratio, err_sum, diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py index 36ab8c33..e942d5a0 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py @@ -10,11 +10,12 @@ import pandas as pd from ctapipe.containers import EventType from ctapipe_io_nectarcam.containers import NectarCAMDataContainer -from test_tools_components import PedestalTool -from utils import adc_to_pe, pe2photons, photons2pe from nectarchain.makers.calibration import PedestalNectarCAMCalibrationTool +from .tools_components import PedestalTool +from .utils import adc_to_pe, pe2photons, photons2pe + def get_args(): """ diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py index 637d5de2..6b4393ce 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py @@ -5,7 +5,8 @@ import matplotlib.pyplot as plt import numpy as np -from test_tools_components import ToMPairsTool + +from .tools_components import ToMPairsTool def get_args(): diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py index 7c32b623..1c7e57eb 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py @@ -7,8 +7,9 @@ import matplotlib.pyplot as plt import numpy as np -from test_tools_components import TimingResolutionTestTool -from utils import pe2photons, photons2pe + +from .tools_components import TimingResolutionTestTool +from .utils import pe2photons, photons2pe def get_args(): diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py index e22f575e..df87d7de 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py @@ -16,13 +16,14 @@ from lmfit.models import Model from scipy.interpolate import InterpolatedUnivariateSpline from scipy.signal import find_peaks -from utils import adc_to_pe, argmedian from nectarchain.data.container import NectarCAMContainer from nectarchain.makers import EventsLoopNectarCAMCalibrationTool from nectarchain.makers.component import NectarCAMComponent from nectarchain.utils import ComponentUtils +from .utils import adc_to_pe, argmedian + # overriding so we can have maxevents in the path def _init_output_path(self): diff --git a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py index 819402ff..cf0e7222 100644 --- a/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py @@ -8,8 +8,9 @@ import matplotlib.pyplot as plt import numpy as np -from test_tools_components import TriggerTimingTestTool -from utils import pe2photons + +from .tools_components import TriggerTimingTestTool +from .utils import pe2photons def get_args(): From 2dfdd82126be344c25bb3a198c78867ef49cc4e9 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 7 Jan 2025 18:49:20 +0100 Subject: [PATCH 11/36] bugfix : logger improvement split_run method --- src/nectarchain/makers/core.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 703a1ac2..8c12bed8 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -16,6 +16,7 @@ classes_with_traits, flag, ) +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer from ctapipe.io import HDF5TableWriter from ctapipe.io.datawriter import DATA_MODEL_VERSION from tables.exceptions import HDF5ExtError @@ -23,7 +24,7 @@ from traitlets import default from ..data import DataManagement -from ..data.container import LightNectarCAMEventSource +from ctapipe_io_nectarcam import LightNectarCAMEventSource from ..data.container.core import NectarCAMContainer, TriggerMapContainer from ..utils import ComponentUtils from .component import NectarCAMComponent, get_valid_component @@ -433,7 +434,7 @@ def start( self._setup_components() n_events_in_slice = 0 - def split_run(self, n_events_in_slice, event): + def split_run(self, n_events_in_slice : int = None, event : NectarCAMDataContainer = None): """Method to decide if criteria to end a run slice are met""" condition = ( self.events_per_slice is not None @@ -487,7 +488,8 @@ def _write_container(self, container: Container, index_component: int = 0) -> No log.warning(e, exc_info=True) self.log.warning("the container has not been written") except Exception as e: - self.log.error(e, exc_info=True) + log.error(e, exc_info=True) + self.log.error(e.args[0], exc_info=True) raise e @property @@ -550,7 +552,7 @@ class DelimiterLoopNectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def split_run(self, n_events_in_slice, event): + def split_run(self, n_events_in_slice : int = None, event : NectarCAMDataContainer = None): """Method to decide if criteria to end a run slice is met""" condition = event.trigger.event_type == EventType.UNKNOWN return condition From c5d4419e41fbfa82a021310418931d167285d91b Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 7 Jan 2025 18:50:05 +0100 Subject: [PATCH 12/36] static method decorator were missing --- src/nectarchain/data/management.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index 7564801e..50914c51 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -262,12 +262,12 @@ def __get_GRID_location_ELog( return lfns else: return url_data - + @staticmethod def find_waveforms(run_number, max_events=None): return __class__.__find_computed_data( run_number=run_number, max_events=max_events, data_type="waveforms" ) - + @staticmethod def find_charges( run_number, method="FullWaveformSum", str_extractor_kwargs="", max_events=None ): @@ -277,7 +277,7 @@ def find_charges( ext=f"_{method}_{str_extractor_kwargs}.h5", data_type="charges", ) - + @staticmethod def find_photostat( FF_run_number, ped_run_number, @@ -296,7 +296,7 @@ def find_photostat( if len(full_file) != 1: raise Exception(f"the files is {full_file}") return full_file - + @staticmethod def find_SPE_combined( run_number, method="FullWaveformSum", str_extractor_kwargs="" ): @@ -306,7 +306,7 @@ def find_SPE_combined( str_extractor_kwargs=str_extractor_kwargs, keyword="FlatFieldCombined", ) - + @staticmethod def find_SPE_nominal( run_number, method="FullWaveformSum", str_extractor_kwargs="", free_pp_n=False ): @@ -317,7 +317,7 @@ def find_SPE_nominal( free_pp_n=free_pp_n, keyword="FlatFieldSPENominal", ) - + @staticmethod def find_SPE_HHV( run_number, method="FullWaveformSum", @@ -357,7 +357,7 @@ def find_SPE_HHV( return [all_files[index]] else: return full_file - + @staticmethod def __find_computed_data( run_number, max_events=None, ext=".h5", data_type="waveforms" ): From 404f189a5a7f17bd30bfcd221a83f0fed0fefed9 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 7 Jan 2025 18:50:36 +0100 Subject: [PATCH 13/36] bugfix test containers --- src/nectarchain/data/container/tests/test_charge.py | 2 +- src/nectarchain/data/container/tests/test_core.py | 4 ++-- src/nectarchain/data/container/tests/test_gain.py | 4 ++-- src/nectarchain/data/container/tests/test_waveform.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nectarchain/data/container/tests/test_charge.py b/src/nectarchain/data/container/tests/test_charge.py index 3d146b7b..2c902b1c 100644 --- a/src/nectarchain/data/container/tests/test_charge.py +++ b/src/nectarchain/data/container/tests/test_charge.py @@ -131,7 +131,7 @@ def test_from_hdf5(self, tmp_path="/tmp"): mode="w", group_name="data", ) - writer.write(table_name="ChargesContainer", containers=charge_container) + writer.write(table_name="ChargesContainer_0", containers=charge_container) writer.close() loaded_charge_container = next(ChargesContainer.from_hdf5(tmp_path)) diff --git a/src/nectarchain/data/container/tests/test_core.py b/src/nectarchain/data/container/tests/test_core.py index 10513a04..61603bb9 100644 --- a/src/nectarchain/data/container/tests/test_core.py +++ b/src/nectarchain/data/container/tests/test_core.py @@ -120,9 +120,9 @@ def test_from_hdf5(self, tmp_path="/tmp"): mode="w", group_name="data", ) - writer.write(table_name="ArrayDataContainer", containers=arrayDataContainer) + writer.write(table_name="ArrayDataContainer_0", containers=arrayDataContainer) writer.close() - loaded_arrayDataContainer = next(arrayDataContainer.from_hdf5(tmp_path)) + loaded_arrayDataContainer = next(ArrayDataContainer.from_hdf5(tmp_path)) assert isinstance(loaded_arrayDataContainer, ArrayDataContainer) assert loaded_arrayDataContainer.run_number == arrayDataContainer.run_number assert ( diff --git a/src/nectarchain/data/container/tests/test_gain.py b/src/nectarchain/data/container/tests/test_gain.py index e6a10b62..31d208ec 100644 --- a/src/nectarchain/data/container/tests/test_gain.py +++ b/src/nectarchain/data/container/tests/test_gain.py @@ -83,7 +83,7 @@ def test_from_hdf5(self, tmp_path="/tmp"): mode="w", group_name="data", ) - writer.write(table_name="GainContainer", containers=gain_container) + writer.write(table_name="GainContainer_0", containers=gain_container) writer.close() loaded_gain_container = next(GainContainer.from_hdf5(tmp_path)) @@ -172,7 +172,7 @@ def test_from_hdf5(self, tmp_path="/tmp"): mode="w", group_name="data", ) - writer.write(table_name="SPEfitContainer", containers=SPEfit_container) + writer.write(table_name="SPEfitContainer_0", containers=SPEfit_container) writer.close() loaded_SPEfit_container = next(SPEfitContainer.from_hdf5(tmp_path)) diff --git a/src/nectarchain/data/container/tests/test_waveform.py b/src/nectarchain/data/container/tests/test_waveform.py index 54851e36..f240c51f 100644 --- a/src/nectarchain/data/container/tests/test_waveform.py +++ b/src/nectarchain/data/container/tests/test_waveform.py @@ -112,7 +112,7 @@ def test_from_hdf5(self, tmp_path="/tmp"): mode="w", group_name="data", ) - writer.write(table_name="WaveformsContainer", containers=waveform_container) + writer.write(table_name="WaveformsContainer_0", containers=waveform_container) writer.close() loaded_waveform_container = next(WaveformsContainer.from_hdf5(tmp_path)) From 8c2ac418eeee4ad02cd96a30eec2d0abceac4c5c Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 7 Jan 2025 18:52:53 +0100 Subject: [PATCH 14/36] unit test fore nectarchain.makers .core --- src/nectarchain/makers/tests/test_core.py | 396 +++++++++++++++++++--- 1 file changed, 353 insertions(+), 43 deletions(-) diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core.py index 341c45f8..602e3c6f 100644 --- a/src/nectarchain/makers/tests/test_core.py +++ b/src/nectarchain/makers/tests/test_core.py @@ -1,43 +1,353 @@ -# import logging -# -# from nectarchain.data.container import ChargesContainer -# from nectarchain.makers import ArrayDataMaker, ChargesMaker, WaveformsMaker -# -# logging.basicConfig( -# format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG -# ) -# log = logging.getLogger(__name__) -# log.handlers = logging.getLogger("__main__").handlers -# -# -# class TestArrayDataMaker: -# run_number = 3938 -# max_events = 100 -# -# def test_merge(self): -# chargesMaker = ChargesMaker( -# run_number=TestArrayDataMaker.run_number, -# max_events=TestArrayDataMaker.max_events, -# ) -# charges_1 = chargesMaker.make() -# chargesMaker_2 = ChargesMaker( -# run_number=TestArrayDataMaker.run_number, -# max_events=TestArrayDataMaker.max_events, -# ) -# charges_2 = chargesMaker_2.make() -# -# merged = ArrayDataMaker.merge(charges_1, charges_2) -# assert isinstance(merged, ChargesContainer) -# -# def test_merge_different_container(self): -# chargesMaker = ChargesMaker( -# run_number=TestArrayDataMaker.run_number, -# max_events=TestArrayDataMaker.max_events, -# ) -# charges_1 = chargesMaker.make() -# wfsMaker_2 = WaveformsMaker( -# run_number=TestArrayDataMaker.run_number, -# max_events=TestArrayDataMaker.max_events, -# ) -# wfs_2 = wfsMaker_2.make() -# merged = ArrayDataMaker.merge(charges_1, wfs_2) +import logging +from ctapipe.utils import get_dataset_path +from nectarchain.makers.core import ( + BaseNectarCAMCalibrationTool, + EventsLoopNectarCAMCalibrationTool, + DelimiterLoopNectarCAMCalibrationTool + ) +from ctapipe.containers import EventType +from nectarchain.data import NectarCAMContainer +from nectarchain.makers.component import NectarCAMComponent +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer +from ctapipe_io_nectarcam import LightNectarCAMEventSource +import os +import pathlib +import pytest +from unittest.mock import patch +import numpy as np +import traitlets +from ctapipe.core.container import Container +from nectarchain.data.container.core import ( + NectarCAMContainer, + TriggerMapContainer, +) +from ctapipe.containers import TriggerContainer +from unittest.mock import MagicMock + + +logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG +) +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + +class TestBaseNectarCAMCalibrationTool: + RUN_NUMBER = 3938 + RUN_FILE = get_dataset_path("NectarCAM.Run3938.30events.fits.fz") + + def test_load_run(self): + eventsource = BaseNectarCAMCalibrationTool.load_run(run_number = self.RUN_NUMBER,max_events=1,run_file = self.RUN_FILE) + assert isinstance(eventsource,LightNectarCAMEventSource) + +class MockComponent(NectarCAMComponent): + def __init__(self, *args, **kwargs): + pass + def __call__(self, event, *args, **kwargs): + pass + def start(self): + pass + def finish(self): + return [NectarCAMContainer()] + +class TestEventsLoopNectarCAMCalibrationTool(TestBaseNectarCAMCalibrationTool): + MAX_EVENTS = 10 + EVENTS_PER_SLICE = 8 + + + + @pytest.fixture + def tool_instance(self): + return EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, + ) + @pytest.fixture + def tool_instance_run_file(self): + return EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, + run_file=self.RUN_FILE, + ) + + def test_init_output_path(self, tool_instance): + expected_path = pathlib.Path( + f"{os.environ.get('NECTARCAMDATA', '/tmp')}/runs/EventsLoopNectarCAMCalibration_run{self.RUN_NUMBER}.h5" + ) + assert tool_instance.output_path == expected_path + assert tool_instance.run_number == self.RUN_NUMBER + assert tool_instance.max_events is None + assert tool_instance.run_file is None + assert tool_instance.name == "EventsLoopNectarCAMCalibration" + assert tool_instance.events_per_slice is None + + def test_init_with_output_path(self): + custom_path = pathlib.Path("/custom/path/output.h5") + tool_instance = EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER, output_path=custom_path) + assert tool_instance.output_path == custom_path + + def test_init_with_max_events(self): + tool_instance = EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER, max_events = 10) + assert tool_instance.max_events == self.MAX_EVENTS + + def test_init_with_events_per_slice(self): + tool_instance = EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER, events_per_slice = 4) + assert tool_instance.events_per_slice == self.EVENTS_PER_SLICE + + def test_init_with_run_file(self): + tool_instance = EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER, run_file = self.RUN_FILE) + assert tool_instance.run_file == self.RUN_FILE + + def test_load_eventsource(self, tool_instance_run_file): + tool_instance_run_file._load_eventsource() + assert isinstance(tool_instance_run_file.event_source, LightNectarCAMEventSource) + assert tool_instance_run_file.event_source.input_url == self.RUN_FILE + def test_load_eventsource_max_events(self, tool_instance_run_file): + tool_instance_run_file.max_events = self.MAX_EVENTS + tool_instance_run_file._load_eventsource() + assert isinstance(tool_instance_run_file.event_source, LightNectarCAMEventSource) + assert tool_instance_run_file.event_source.input_url == self.RUN_FILE + assert tool_instance_run_file.event_source.max_events == self.MAX_EVENTS + + @patch("nectarchain.makers.core.HDF5TableWriter") + @patch("nectarchain.makers.core.os.remove") + @patch("nectarchain.makers.core.os.makedirs") + def test_init_writer_full_mode(self, mock_makedirs, mock_remove, mock_writer, tool_instance): + tool_instance.overwrite = True + tool_instance._init_writer(sliced=False) + mock_remove.assert_called_once_with(tool_instance.output_path) + mock_makedirs.assert_called_once_with(tool_instance.output_path.parent, exist_ok=True) + mock_writer.assert_called_once_with( + filename=tool_instance.output_path, + parent=tool_instance, + mode="w", + group_name="data", + ) + + + + @patch("nectarchain.makers.core.HDF5TableWriter") + @patch("nectarchain.makers.core.os.remove") + @patch("nectarchain.makers.core.os.makedirs") + def test_init_writer_sliced_mode(self, mock_makedirs, mock_remove, mock_writer, tool_instance): + tool_instance.overwrite = True + tool_instance._init_writer(sliced=True, slice_index=1) + mock_remove.assert_not_called() + mock_makedirs.assert_called_once_with(tool_instance.output_path.parent, exist_ok=True) + mock_writer.assert_called_once_with( + filename=tool_instance.output_path, + parent=tool_instance, + mode="a", + group_name="data_1", + ) + @patch("nectarchain.makers.core.HDF5TableWriter") + @patch("nectarchain.makers.core.os.remove") + @patch("nectarchain.makers.core.os.makedirs") + def test_init_writer_overwrite_false(self, mock_makedirs, mock_remove, mock_writer, tool_instance): + tool_instance.overwrite = False + with patch("nectarchain.makers.core.os.path.exists", return_value=True): + with pytest.raises(Exception): + tool_instance._init_writer(sliced=False) + mock_remove.assert_not_called() + mock_makedirs.assert_not_called() + mock_writer.assert_not_called() + + def test_setup_eventsource(self, tool_instance_run_file): + tool_instance_run_file._setup_eventsource() + + assert tool_instance_run_file._npixels == tool_instance_run_file.event_source.nectarcam_service.num_pixels + assert np.all(tool_instance_run_file._pixels_id == tool_instance_run_file.event_source.nectarcam_service.pixel_ids) + assert isinstance(tool_instance_run_file.event_source,LightNectarCAMEventSource) + + + @patch("nectarchain.makers.core.ComponentUtils.get_class_name_from_ComponentName", return_value="ValidComponentClass") + @patch("nectarchain.makers.core.ComponentUtils.get_configurable_traits", return_value={'trait1': 'value1'}) + def test_get_provided_component_kwargs(self,mock_get_class_name,mock_get_valid_component,tool_instance): + tool_instance.trait1 = "value1" + output_component_kwargs = tool_instance._get_provided_component_kwargs('componentName') + assert output_component_kwargs == {"trait1": "value1"} + + + @patch("nectarchain.makers.core.Component") + @patch("nectarchain.makers.core.get_valid_component", return_value=["WaveformsComponent"]) + @patch("nectarchain.makers.core.ComponentUtils.get_class_name_from_ComponentName", return_value="WaveformsComponentClass") + @patch("nectarchain.makers.core.ComponentUtils.get_configurable_traits", return_value={"trait1": "value1"}) + def test_setup_components(self, mock_get_configurable_traits, mock_get_class_name, mock_get_valid_component, mock_component, tool_instance_run_file): + with pytest.raises(traitlets.traitlets.TraitError): + tool_instance_run_file.componentsList = ["ValidComponent"] + tool_instance_run_file.componentsList = ["WaveformsComponent"] + tool_instance_run_file.trait1 = "value1" + tool_instance_run_file._setup_eventsource() + tool_instance_run_file._setup_components() + mock_get_valid_component.assert_called_once() + mock_get_class_name.assert_called_once_with("WaveformsComponent") + mock_get_configurable_traits.assert_called_once_with("WaveformsComponentClass") + mock_component.from_name.assert_called_once_with( + "WaveformsComponent", + subarray=tool_instance_run_file.event_source.subarray, + parent=tool_instance_run_file, + trait1="value1" + ) + assert len(tool_instance_run_file.components) == 1 + assert tool_instance_run_file.components[0] == mock_component.from_name.return_value + + + @patch("nectarchain.makers.core.os.remove") + @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_eventsource") + @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_components") + @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._init_writer") + def test_setup(self, mock_init_writer, mock_setup_components, mock_setup_eventsource, mock_remove, tool_instance_run_file): + tool_instance_run_file.overwrite = True + tool_instance_run_file.output_path = pathlib.Path("/tmp/test_output.h5") + + with patch("nectarchain.makers.core.pathlib.Path.exists", return_value=True): + tool_instance_run_file.setup() + + mock_setup_eventsource.assert_called_once() + mock_setup_components.assert_called_once() + mock_remove.assert_called_once_with(tool_instance_run_file.output_path) + mock_init_writer.assert_called_once_with(sliced=False, slice_index=1) + assert tool_instance_run_file._n_traited_events == 0 + + + def test_setup_run_number_not_set(self, tool_instance): + tool_instance.run_number = -1 + with pytest.raises(Exception, match="run_number need to be set up"): + tool_instance.setup() + + def test_split_run(self,tool_instance): + event = NectarCAMDataContainer() + assert not(tool_instance.split_run(n_events_in_slice = 6,event=event)) + tool_instance.events_per_slice = 4 + assert tool_instance.split_run(n_events_in_slice = 6,event = event) + assert not(tool_instance.split_run(n_events_in_slice =2,event = event)) + + @patch("nectarchain.makers.core.Component") + def test_start(self,mock_component,tool_instance_run_file): + tool_instance_run_file.overwrite = True + tool_instance_run_file.setup() + n_events = len(tool_instance_run_file.event_source) + tool_instance_run_file.components = [mock_component.from_name.return_value] + tool_instance_run_file.start() + assert tool_instance_run_file._n_traited_events == n_events + + @patch("nectarchain.makers.core.Component") + def test_start_n_events(self,mock_component,tool_instance_run_file): + tool_instance_run_file.overwrite = True + tool_instance_run_file.setup() + tool_instance_run_file.components = [mock_component.from_name.return_value] + tool_instance_run_file.start(n_events = 10) + assert tool_instance_run_file._n_traited_events == 10 + + @patch("nectarchain.makers.core.Component") + @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components") + @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_components") + def test_start_sliced(self,mock_setup_components, mock_finish_components,mock_component,tool_instance_run_file): + tool_instance_run_file.overwrite = True + tool_instance_run_file.events_per_slice = self.EVENTS_PER_SLICE + tool_instance_run_file.setup() + n_events = len(tool_instance_run_file.event_source) + tool_instance_run_file.components = [MockComponent()] + tool_instance_run_file.start() + assert mock_finish_components.call_count == n_events//self.EVENTS_PER_SLICE + assert mock_setup_components.call_count == n_events//self.EVENTS_PER_SLICE + 1 + + @patch("nectarchain.makers.core.Component") + @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components") + def test_finish(self, mock_finish_components, mock_component, tool_instance_run_file): + tool_instance_run_file.overwrite = True + tool_instance_run_file.setup() + tool_instance_run_file.components = [mock_component.from_name.return_value] + mock_finish_components.return_value = ["output"] + + output = tool_instance_run_file.finish() + + mock_finish_components.assert_called_once() + assert output is None + assert tool_instance_run_file.writer.h5file.isopen == 0 + @patch("nectarchain.makers.core.Component") + @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components") + def test_finish_with_output(self, mock_finish_components, mock_component, tool_instance_run_file): + tool_instance_run_file.overwrite = True + tool_instance_run_file.setup() + tool_instance_run_file.components = [mock_component.from_name.return_value] + + output = tool_instance_run_file.finish(return_output_component=True) + + mock_finish_components.assert_called_once() + assert output is not None + assert tool_instance_run_file.writer.h5file.isopen == 0 + @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components") + def test_finish_components(self, mock_finish_components, tool_instance_run_file): + tool_instance_run_file.overwrite = True + tool_instance_run_file.setup() + tool_instance_run_file.components = [MockComponent()] + + output = tool_instance_run_file._finish_components() + + assert mock_finish_components.called_with([MockComponent().finish()], 0) + + @patch("nectarchain.makers.core.HDF5TableWriter") + def test_write_container_with_nectarcam_container(self, mock_writer, tool_instance_run_file): + tool_instance_run_file.writer = mock_writer + container = MagicMock(spec=NectarCAMContainer) + container.validate = MagicMock() + tool_instance_run_file._write_container(container, index_component=0) + container.validate.assert_called_once() + mock_writer.write.assert_called_once_with( + table_name=f"{container.__class__.__name__}_0", + containers=container, + ) + + @patch("nectarchain.makers.core.HDF5TableWriter") + def test_write_container_with_triggermap_container(self, mock_writer, tool_instance_run_file): + tool_instance_run_file.writer = mock_writer + container = MagicMock(spec=TriggerMapContainer) + container.validate = MagicMock() + container.containers = { + EventType.FLATFIELD: MagicMock(spec=Container), + EventType.UNKNOWN: MagicMock(spec=Container), + } + tool_instance_run_file._write_container(container, index_component=0) + container.validate.assert_called_once() + mock_writer.write.assert_any_call( + table_name=f"{container.containers[EventType.FLATFIELD].__class__.__name__}_0/{EventType.FLATFIELD.name}", + containers=container.containers[EventType.FLATFIELD], + ) + mock_writer.write.assert_any_call( + table_name=f"{container.containers[EventType.UNKNOWN].__class__.__name__}_0/{EventType.UNKNOWN.name}", + containers=container.containers[EventType.UNKNOWN], + ) + + @patch("nectarchain.makers.core.HDF5TableWriter") + def test_write_container_with_invalid_container(self, mock_writer, tool_instance_run_file): + tool_instance_run_file.writer = mock_writer + container = MagicMock(spec=Container) + container.validate = MagicMock() + with pytest.raises(TypeError, match="component output must be an instance of TriggerMapContainer or NectarCAMContainer"): + tool_instance_run_file._write_container(container, index_component=0) + container.validate.assert_called_once() + mock_writer.write.assert_not_called() + + + @patch("nectarchain.makers.core.HDF5TableWriter") + def test_write_container_with_generic_exception(self, mock_writer, tool_instance_run_file): + tool_instance_run_file.writer = mock_writer + container = MagicMock(spec=NectarCAMContainer) + container.validate = MagicMock(side_effect=Exception("Generic error")) + with patch.object(tool_instance_run_file.log, 'error') as mock_log_error: + with pytest.raises(Exception, match="Generic error"): + tool_instance_run_file._write_container(container, index_component=0) + container.validate.assert_called_once() + mock_writer.write.assert_not_called() + mock_log_error.assert_called_with("Generic error", exc_info=True) + + + +class TestDelimiterLoopNectarCAMCalibrationTool: + def test_split_run(self) : + tool = DelimiterLoopNectarCAMCalibrationTool() + event = NectarCAMDataContainer(trigger = TriggerContainer(event_type = EventType.FLATFIELD)) + assert not(tool.split_run(event = event)) + event = NectarCAMDataContainer(trigger = TriggerContainer(event_type = EventType.UNKNOWN)) + assert tool.split_run(event = event) + + From 413c9b41efa130e824f612cbb801879c8b87d874 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 16:40:43 +0100 Subject: [PATCH 15/36] bugfix with new version of LIghtNectarCAMEventSource from ctapipe_io_nectarcam --- src/nectarchain/makers/component/core.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/nectarchain/makers/component/core.py b/src/nectarchain/makers/component/core.py index 5e36e71f..eb9f2072 100644 --- a/src/nectarchain/makers/component/core.py +++ b/src/nectarchain/makers/component/core.py @@ -10,6 +10,7 @@ from ctapipe.instrument import CameraGeometry from ctapipe_io_nectarcam import constants from ctapipe_io_nectarcam.containers import NectarCAMDataContainer +from ctapipe_io_nectarcam.constants import N_PIXELS from ...data.container.core import ArrayDataContainer @@ -225,9 +226,14 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.__ucts_event_counter[f"{name}"].append( event.nectarcam.tel[__class__.TEL_ID.default_value].evt.ucts_event_counter ) - self.__trig_patter_all[f"{name}"].append( - event.nectarcam.tel[__class__.TEL_ID.default_value].evt.trigger_pattern.T + if event.nectarcam.tel[__class__.TEL_ID.default_value].evt.trigger_pattern is None : + self.__trig_patter_all[f"{name}"].append( + np.empty((4,N_PIXELS)).T ) + else : + self.__trig_patter_all[f"{name}"].append( + event.nectarcam.tel[__class__.TEL_ID.default_value].evt.trigger_pattern.T + ) if kwargs.get("return_wfs", False): get_wfs_hg = event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id] From f0236953c2d854f6e461cad76843a9d85e608ef2 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 16:41:06 +0100 Subject: [PATCH 16/36] very small code improvement --- src/nectarchain/makers/calibration/tests/test_pedestal_tool.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py index d5f382dc..baea50c7 100644 --- a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py +++ b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py @@ -32,8 +32,7 @@ def test_base(self): expected_ucts_timestamp_min = [1674462932637854793, 1715007113924900896] expected_ucts_timestamp_max = [1674462932695877994, 1715007123524920096] - for i, run in enumerate(runs["Run number"]): - run_number = runs["Run number"][i] + for i, run_number in enumerate(runs["Run number"]): run_file = runs["Run file"][i] n_pixels = runs["N pixels"][i] with tempfile.TemporaryDirectory() as tmpdirname: From 37141c2b995a4e780d9d6d126f23729384201c3e Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 16:41:26 +0100 Subject: [PATCH 17/36] Unit test for WaveformsNectarCAMCalibrationTool --- .../makers/tests/test_waveforms_makers.py | 341 +++++++++--------- 1 file changed, 175 insertions(+), 166 deletions(-) diff --git a/src/nectarchain/makers/tests/test_waveforms_makers.py b/src/nectarchain/makers/tests/test_waveforms_makers.py index 4ac32b38..18a14439 100644 --- a/src/nectarchain/makers/tests/test_waveforms_makers.py +++ b/src/nectarchain/makers/tests/test_waveforms_makers.py @@ -1,166 +1,175 @@ -# import logging -# -# import numpy as np -# import pytest -# from ctapipe.containers import EventType -# -# from nectarchain.data.container import WaveformsContainer, WaveformsContainerIO -# from nectarchain.makers.waveforms_makers import WaveformsMaker -# -# logging.basicConfig( -# format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG -# ) -# log = logging.getLogger(__name__) -# log.handlers = logging.getLogger("__main__").handlers -# -# -# @pytest.disable() -# class TestWaveformsMaker: -# run_number = 3938 -# max_events = 100 -# -# def test_instance(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# assert isinstance(waveformsMaker, WaveformsMaker) -# -# def test_shape_valid(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer = waveformsMaker.make()[0] -# -# assert waveformsContainer.nevents <= TestWaveformsMaker.max_events -# assert waveformsContainer.run_number == TestWaveformsMaker.run_number -# assert (waveformsContainer.ucts_timestamp.shape == -# (waveformsContainer.nevents,)) -# assert waveformsContainer.ucts_busy_counter.shape == ( -# waveformsContainer.nevents, -# ) -# assert waveformsContainer.ucts_event_counter.shape == ( -# waveformsContainer.nevents, -# ) -# assert waveformsContainer.event_type.shape == (waveformsContainer.nevents,) -# assert waveformsContainer.event_id.shape == (waveformsContainer.nevents,) -# assert ( -# waveformsContainer.trig_pattern_all.shape[0] == waveformsContainer.nevents -# ) -# assert waveformsContainer.trig_pattern_all.shape[2] == 4 -# -# assert waveformsContainer.trig_pattern.shape[0] == waveformsContainer.nevents -# assert waveformsContainer.multiplicity.shape == (waveformsContainer.nevents,) -# -# assert waveformsContainer.wfs_hg.mean() != 0 -# assert waveformsContainer.wfs_lg.mean() != 0 -# assert waveformsContainer.wfs_hg.shape == ( -# waveformsContainer.nevents, -# waveformsContainer.npixels, -# waveformsContainer.nsamples, -# ) -# assert waveformsContainer.wfs_lg.shape == ( -# waveformsContainer.nevents, -# waveformsContainer.npixels, -# waveformsContainer.nsamples, -# ) -# assert waveformsContainer.broken_pixels_hg.shape == ( -# waveformsContainer.nevents, -# waveformsContainer.npixels, -# ) -# assert waveformsContainer.broken_pixels_lg.shape == ( -# waveformsContainer.nevents, -# waveformsContainer.npixels, -# ) -# -# def test_all_multiple_trigger(self): -# trigger1 = EventType.FLATFIELD -# trigger2 = EventType.SKY_PEDESTAL -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make( -# trigger_type=[trigger1, trigger2], restart_from_begining=True -# ) -# for waveformsContainer in waveformsContainer_list: -# assert isinstance(waveformsContainer, WaveformsContainer) -# assert waveformsContainer.wfs_hg.mean() != 0 -# -# def test_all_trigger_None(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make() -# assert isinstance(waveformsContainer_list[0], WaveformsContainer) -# -# def test_select_waveforms_hg(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make() -# pixel_id = np.array([3, 67, 87]) -# assert isinstance( -# WaveformsMaker.select_waveforms_hg(waveformsContainer_list[0], pixel_id), -# np.ndarray, -# ) -# assert isinstance( -# WaveformsMaker.select_waveforms_lg(waveformsContainer_list[0], pixel_id), -# np.ndarray, -# ) -# -# def test_sort_WaveformsContainer(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make() -# sortWfs = WaveformsMaker.sort(waveformsContainer_list[0], method="event_id") -# assert np.array_equal( -# sortWfs.event_id, np.sort(waveformsContainer_list[0].event_id) -# ) -# -# def test_write_load_container(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make() -# WaveformsContainerIO.write( -# "/tmp/test_wfs_container/", waveformsContainer_list[0], overwrite=True -# ) -# loaded_wfs = WaveformsContainerIO.load( -# "/tmp/test_wfs_container", TestWaveformsMaker.run_number -# ) -# assert np.array_equal(waveformsContainer_list[0].wfs_hg, loaded_wfs.wfs_hg) -# -# def test_create_from_events_list(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# events_list = [] -# for i, event in enumerate(waveformsMaker._reader): -# events_list.append(event) -# waveformsContainer = WaveformsMaker.create_from_events_list( -# events_list, -# waveformsMaker.run_number, -# waveformsMaker.npixels, -# waveformsMaker.nsamples, -# waveformsMaker.subarray, -# waveformsMaker.pixels_id, -# ) -# assert isinstance(waveformsContainer, WaveformsContainer) -# -# -# if __name__ == "__main__": -# import logging -# -# logging.basicConfig( -# format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG -# ) -# log = logging.getLogger(__name__) -# log.handlers = logging.getLogger("__main__").handlers +import logging +import tempfile +import numpy as np +import pytest +from ctapipe.containers import EventType +from pathlib import Path +from nectarchain.data.container import WaveformsContainer, WaveformsContainers +from nectarchain.makers import WaveformsNectarCAMCalibrationTool +from ctapipe.utils import get_dataset_path +from ctapipe_io_nectarcam.constants import N_SAMPLES + +"This test file test the overall workflow of the WaveformsNectarCAMCalibrationTool, covering also the EventsLoopNectarCAMCalibrationTool, which cannot be tested on data because the ArrayDataComponent within the WaveformsNectarCAMCalibrationTool is an abstract Component. However the EventsLoopNectarCAMCalibrationTool is still covered by unit test using pytest patches to mock the Component behavior." + + +class TestWaveformsNectarCAMCalibrationTool : + RUNS = { + "Run number": [3938, 5288], + "Run file": [ + get_dataset_path("NectarCAM.Run3938.30events.fits.fz"), + get_dataset_path("NectarCAM.Run5288.0001.fits.fz"), + ], + "nevents" : [30,13], + "N pixels": [1834, 1848], + "eventType" : EventType.SKY_PEDESTAL, + "wfs_lg_min" : [230,233], + "wfs_lg_max" : [264,269], + "wfs_lg_mean" : [2468,2509], + "wfs_lg_std" : [33,36], + "wfs_hg_min" : [178,195], + "wfs_hg_max" : [288,298], + "wfs_hg_mean" : [2467,2508], + "wfs_hg_std" : [36,38], + "expected_ucts_timestamp_min" : [1674462932637854793, 1715007113924900896], + "expected_ucts_timestamp_max" : [1674462932695877994, 1715007123524920096], +} + OVERWRITE = True + + def general_structure_testing(self,output : WaveformsContainer,nevents:int,n_pixels:int,run_number:int) : + assert isinstance(output.pixels_id, np.ndarray) + assert output.pixels_id.dtype == np.uint16 + assert np.shape(output.pixels_id) == (n_pixels,) + assert output.run_number == run_number + assert output.camera == 'NectarCam-003' + assert output.npixels == n_pixels + assert isinstance(output.ucts_busy_counter, np.ndarray) + assert output.ucts_busy_counter.dtype == np.uint32 + assert isinstance(output.ucts_event_counter, np.ndarray) + assert output.ucts_event_counter.dtype == np.uint32 + assert isinstance(output.event_type, np.ndarray) + assert output.event_type.dtype == np.uint8 + assert np.all(output.event_type == self.RUNS["eventType"].value) + assert isinstance(output.trig_pattern, np.ndarray) + assert output.trig_pattern.dtype == bool + assert isinstance(output.trig_pattern_all, np.ndarray) + assert output.trig_pattern_all.dtype == bool + assert isinstance(output.multiplicity, np.ndarray) + assert output.multiplicity.dtype == np.uint16 + assert isinstance(output.ucts_timestamp,np.ndarray) + assert output.ucts_timestamp.dtype == np.uint64 + + assert isinstance(output.event_id,np.ndarray) + assert output.event_id.dtype == np.uint32 + assert isinstance(output.broken_pixels_hg,np.ndarray) + assert output.broken_pixels_hg.dtype == bool + assert output.broken_pixels_hg.shape == (nevents,n_pixels) + assert isinstance(output.broken_pixels_lg,np.ndarray) + assert output.broken_pixels_lg.dtype == bool + assert output.broken_pixels_lg.shape == (nevents,n_pixels) + assert output.wfs_hg.shape == (nevents, n_pixels, N_SAMPLES) + assert output.wfs_lg.shape == (nevents, n_pixels, N_SAMPLES) + assert isinstance(output.wfs_hg, np.ndarray) + assert isinstance(output.wfs_lg, np.ndarray) + assert output.wfs_hg.dtype == np.uint16 + assert output.wfs_lg.dtype == np.uint16 + + + def test_base(self): + """ + Test basic functionality, including IO on disk + """ + + events_per_slice =[None,None,10,11,8] + max_events = [None,10,None,None,10] + + + for _max_events,_events_per_slice in zip(max_events,events_per_slice): + for i, run_number in enumerate(self.RUNS["Run number"]): + run_file = self.RUNS["Run file"][i] + n_pixels = self.RUNS["N pixels"][i] + with tempfile.TemporaryDirectory() as tmpdirname: + outfile = tmpdirname + "/waveforms.h5" + + # run tool + tool = WaveformsNectarCAMCalibrationTool( + run_number=run_number, + run_file=run_file, + max_events=_max_events, + events_per_slice=_events_per_slice, + log_level=0, + output_path=outfile, + overwrite=self.OVERWRITE, + ) + + tool.setup() + nevents = len(tool.event_source) + assert nevents == self.RUNS["nevents"][i] if _max_events is None else _max_events + tool.start() + output_containers = tool.finish(return_output_component=True)[0] + assert isinstance(output_containers, WaveformsContainers) + output = output_containers.containers[self.RUNS["eventType"]] + assert isinstance(output, WaveformsContainer) + # Check output in memory + if _events_per_slice is not None and nevents%_events_per_slice == 0 : + assert output.nevents is None + else : + assert output.nsamples == N_SAMPLES + if _events_per_slice is None : + assert output.nevents == nevents #nevents has been validated before + else : + assert output.nevents == nevents%_events_per_slice + + self.general_structure_testing( + output, + nevents if _events_per_slice is None else nevents%_events_per_slice, + n_pixels, + run_number + ) + + + if _events_per_slice is None and _max_events is None: + #content only checked for the full run + assert np.min(output.ucts_timestamp) == np.uint64( + self.RUNS["expected_ucts_timestamp_min"][i] + ) + assert np.max(output.ucts_timestamp) == np.uint64( + self.RUNS["expected_ucts_timestamp_max"][i] + ) + assert output.wfs_lg.min() == self.RUNS["wfs_lg_min"][i] + assert output.wfs_lg.max() == self.RUNS["wfs_lg_max"][i] + assert int(10*output.wfs_lg.mean()) == self.RUNS["wfs_lg_mean"][i] + assert int(10*output.wfs_lg.std()) ==self.RUNS["wfs_lg_std"][i] + assert output.wfs_hg.min() == self.RUNS["wfs_hg_min"][i] + assert output.wfs_hg.max() == self.RUNS["wfs_hg_max"][i] + assert int(10*output.wfs_hg.mean()) == self.RUNS["wfs_hg_mean"][i] + assert int(10*output.wfs_hg.std()) ==self.RUNS["wfs_hg_std"][i] + + # Check output on disk + assert Path(outfile).exists() + + waveformsContainers = WaveformsContainers.from_hdf5(outfile) + ncontainers = 0 + for container in waveformsContainers : + ncontainers +=1 + assert isinstance(container, WaveformsContainers) + output = container.containers[self.RUNS["eventType"]] + if _events_per_slice is None : + expected_nevents = nevents + else : + if nevents%_events_per_slice == 0 : + expected_nevents = _events_per_slice + else : + if ncontainers == 1 : + expected_nevents = nevents%_events_per_slice + else : + expected_nevents = _events_per_slice + self.general_structure_testing( + output, + expected_nevents, + n_pixels, + run_number + ) + + assert ncontainers == 1 if _events_per_slice is None else round(nevents / _events_per_slice) + + From 7308cdadd80400b9b2228032f84f7643c19427b9 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 17:13:38 +0100 Subject: [PATCH 18/36] Unit test for the ChargesNectarCAMCalibrationTool --- .../makers/tests/test_charges_makers.py | 382 +++++++++--------- 1 file changed, 184 insertions(+), 198 deletions(-) diff --git a/src/nectarchain/makers/tests/test_charges_makers.py b/src/nectarchain/makers/tests/test_charges_makers.py index 88aa8c82..5978fdf0 100644 --- a/src/nectarchain/makers/tests/test_charges_makers.py +++ b/src/nectarchain/makers/tests/test_charges_makers.py @@ -1,198 +1,184 @@ -# import logging -# -# import numpy as np -# from ctapipe.containers import EventType -# -# from nectarchain.data.container import ChargesContainer, ChargesContainerIO -# from nectarchain.makers import ChargesMaker, WaveformsMaker -# -# logging.basicConfig( -# format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG -# ) -# log = logging.getLogger(__name__) -# log.handlers = logging.getLogger("__main__").handlers -# -# -# class TestChargesMaker: -# run_number = 3938 -# max_events = 100 -# -# def test_instance(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# assert isinstance(chargesMaker, ChargesMaker) -# -# def test_shape_valid(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer = chargesMaker.make()[0] -# -# assert chargesContainer.nevents <= TestChargesMaker.max_events -# assert chargesContainer.run_number == TestChargesMaker.run_number -# assert chargesContainer.ucts_timestamp.shape == (chargesContainer.nevents,) -# assert chargesContainer.ucts_busy_counter.shape == (chargesContainer.nevents,) -# assert (chargesContainer.ucts_event_counter.shape == -# (chargesContainer.nevents,)) -# assert chargesContainer.event_type.shape == (chargesContainer.nevents,) -# assert chargesContainer.event_id.shape == (chargesContainer.nevents,) -# assert chargesContainer.trig_pattern_all.shape[0] == chargesContainer.nevents -# assert chargesContainer.trig_pattern_all.shape[2] == 4 -# -# assert chargesContainer.trig_pattern.shape[0] == chargesContainer.nevents -# assert chargesContainer.multiplicity.shape == (chargesContainer.nevents,) -# -# assert chargesContainer.charges_hg.mean() != 0 -# assert chargesContainer.charges_lg.mean() != 0 -# assert chargesContainer.peak_hg.mean() != 0 -# assert chargesContainer.peak_lg.mean() != 0 -# assert chargesContainer.charges_hg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# assert chargesContainer.charges_lg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# assert chargesContainer.peak_hg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# assert chargesContainer.peak_lg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# -# assert chargesContainer.broken_pixels_hg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# assert chargesContainer.broken_pixels_lg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# -# def test_make_restart_eventsource(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make(restart_from_begining=True) -# assert isinstance(chargesContainer_list[0], ChargesContainer) -# -# def test_make_LocalPeakWindowSum(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make( -# method="LocalPeakWindowSum", window_shift=-4, window_length=16 -# ) -# assert isinstance(chargesContainer_list[0], ChargesContainer) -# -# def test_all_multiple_trigger(self): -# trigger1 = EventType.FLATFIELD -# trigger2 = EventType.SKY_PEDESTAL -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make(trigger_type=[trigger1, trigger2]) -# for chargesContainer in chargesContainer_list: -# assert isinstance(chargesContainer, ChargesContainer) -# -# def test_all_trigger_None(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make() -# assert isinstance(chargesContainer_list[0], ChargesContainer) -# -# def test_create_from_waveforms(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make() -# chargesContainer = ChargesMaker.create_from_waveforms( -# waveformsContainer_list[0], -# method="LocalPeakWindowSum", -# window_shift=-4, -# window_length=16, -# ) -# assert isinstance(chargesContainer, ChargesContainer) -# -# def test_select_charges(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make() -# pixel_id = np.array([3, 67, 87]) -# assert isinstance( -# ChargesMaker.select_charges_hg(chargesContainer_list[0], pixel_id), -# np.ndarray, -# ) -# assert isinstance( -# ChargesMaker.select_charges_lg(chargesContainer_list[0], pixel_id), -# np.ndarray, -# ) -# -# def test_histo(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make() -# histo = ChargesMaker.histo_hg(chargesContainer_list[0]) -# assert isinstance(histo, np.ndarray) -# assert histo.mean() != 0 -# assert histo.shape[0] == 2 -# assert histo.shape[1] == chargesContainer_list[0].npixels -# histo = ChargesMaker.histo_lg(chargesContainer_list[0]) -# assert isinstance(histo, np.ndarray) -# assert histo.mean() != 0 -# assert histo.shape[0] == 2 -# assert histo.shape[1] == chargesContainer_list[0].npixels -# -# def test_sort_ChargesContainer(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make() -# sortWfs = ChargesMaker.sort(chargesContainer_list[0], method="event_id") -# assert np.array_equal( -# sortWfs.event_id, np.sort(chargesContainer_list[0].event_id) -# ) -# -# def test_write_load_container(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make() -# ChargesContainerIO.write( -# "/tmp/test_charge_container/", chargesContainer_list[0], overwrite=True -# ) -# loaded_charge = ChargesContainerIO.load( -# "/tmp/test_charge_container", run_number=TestChargesMaker.run_number -# ) -# assert np.array_equal( -# chargesContainer_list[0].charges_hg, loaded_charge.charges_hg -# ) -# -# -# if __name__ == "__main__": -# import logging -# -# logging.basicConfig( -# format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG -# ) -# log = logging.getLogger(__name__) -# log.handlers = logging.getLogger("__main__").handlers -# TestChargesMaker().test_write_load_container() +import logging +import tempfile +import numpy as np +import pytest +from ctapipe.containers import EventType +from pathlib import Path +from nectarchain.data.container import ChargesContainer, ChargesContainers +from nectarchain.makers import ChargesNectarCAMCalibrationTool +from ctapipe.utils import get_dataset_path + +"This test file test the overall workflow of the ChargesNectarCAMCalibrationTool, adapted from the test of the WaveformsNectarCAMCalibrationTool. There are lot of code duplication but I don't care for now. Do you ?" + + +class TestChargesNectarCAMCalibrationTool : + RUNS = { + "Run number": [3938, 5288], + "Run file": [ + get_dataset_path("NectarCAM.Run3938.30events.fits.fz"), + get_dataset_path("NectarCAM.Run5288.0001.fits.fz"), + ], + "nevents" : [30,13], + "N pixels": [1834, 1848], + "eventType" : EventType.SKY_PEDESTAL, + "charges_lg_min" : [957,973], + "charges_lg_max" : [2059,2113], + "charges_lg_mean" : [19067,19306], + "charges_lg_std" : [2273,2376], + "charges_hg_min" : [963,972], + "charges_hg_max" : [2057,2109], + "charges_hg_mean" : [19106,19275], + "charges_hg_std" : [2139,2357], + "expected_ucts_timestamp_min" : [1674462932637854793, 1715007113924900896], + "expected_ucts_timestamp_max" : [1674462932695877994, 1715007123524920096], +} + OVERWRITE = True + METHOD = "LocalPeakWindowSum" + EXTRACTOR_KWARGS = {"window_shift": 4, "window_width": 8} + + def general_structure_testing(self,output : ChargesContainer,nevents:int,n_pixels:int,run_number:int) : + assert isinstance(output.pixels_id, np.ndarray) + assert output.pixels_id.dtype == np.uint16 + assert np.shape(output.pixels_id) == (n_pixels,) + assert output.run_number == run_number + assert output.camera == 'NectarCam-003' + assert output.npixels == n_pixels + assert isinstance(output.ucts_busy_counter, np.ndarray) + assert output.ucts_busy_counter.dtype == np.uint32 + assert isinstance(output.ucts_event_counter, np.ndarray) + assert output.ucts_event_counter.dtype == np.uint32 + assert isinstance(output.event_type, np.ndarray) + assert output.event_type.dtype == np.uint8 + assert np.all(output.event_type == self.RUNS["eventType"].value) + assert isinstance(output.trig_pattern, np.ndarray) + assert output.trig_pattern.dtype == bool + assert isinstance(output.trig_pattern_all, np.ndarray) + assert output.trig_pattern_all.dtype == bool + assert isinstance(output.multiplicity, np.ndarray) + assert output.multiplicity.dtype == np.uint16 + assert isinstance(output.ucts_timestamp,np.ndarray) + assert output.ucts_timestamp.dtype == np.uint64 + + assert isinstance(output.event_id,np.ndarray) + assert output.event_id.dtype == np.uint32 + assert isinstance(output.broken_pixels_hg,np.ndarray) + assert output.broken_pixels_hg.dtype == bool + assert output.broken_pixels_hg.shape == (nevents,n_pixels) + assert isinstance(output.broken_pixels_lg,np.ndarray) + assert output.broken_pixels_lg.dtype == bool + assert output.broken_pixels_lg.shape == (nevents,n_pixels) + assert output.charges_hg.shape == (nevents, n_pixels) + assert output.charges_lg.shape == (nevents, n_pixels) + assert isinstance(output.charges_hg, np.ndarray) + assert isinstance(output.charges_lg, np.ndarray) + assert output.charges_hg.dtype == np.uint16 + assert output.charges_lg.dtype == np.uint16 + assert output.peak_hg.shape == (nevents, n_pixels) + assert output.peak_lg.shape == (nevents, n_pixels) + assert isinstance(output.peak_hg, np.ndarray) + assert isinstance(output.peak_lg, np.ndarray) + assert output.peak_hg.dtype == np.uint16 + assert output.peak_lg.dtype == np.uint16 + assert isinstance(output.method,str) + assert output.method == self.METHOD + + + def test_base(self): + """ + Test basic functionality, including IO on disk + """ + + events_per_slice =[None,None,10,11,8] + max_events = [None,10,None,None,10] + + + for _max_events,_events_per_slice in zip(max_events,events_per_slice): + for i, run_number in enumerate(self.RUNS["Run number"]): + run_file = self.RUNS["Run file"][i] + n_pixels = self.RUNS["N pixels"][i] + with tempfile.TemporaryDirectory() as tmpdirname: + outfile = tmpdirname + "/charges.h5" + + # run tool + tool = ChargesNectarCAMCalibrationTool( + run_number=run_number, + run_file=run_file, + max_events=_max_events, + events_per_slice=_events_per_slice, + log_level=0, + output_path=outfile, + overwrite=self.OVERWRITE, + method = self.METHOD, + extractor_kwargs = self.EXTRACTOR_KWARGS, + ) + + tool.setup() + nevents = len(tool.event_source) + assert nevents == self.RUNS["nevents"][i] if _max_events is None else _max_events + tool.start() + output_containers = tool.finish(return_output_component=True)[0] + assert isinstance(output_containers, ChargesContainers) + output = output_containers.containers[self.RUNS["eventType"]] + assert isinstance(output, ChargesContainer) + # Check output in memory + if _events_per_slice is not None and nevents%_events_per_slice == 0 : + assert output.nevents is None + else : + if _events_per_slice is None : + assert output.nevents == nevents #nevents has been validated before + else : + assert output.nevents == nevents%_events_per_slice + + self.general_structure_testing( + output, + nevents if _events_per_slice is None else nevents%_events_per_slice, + n_pixels, + run_number + ) + + + if _events_per_slice is None and _max_events is None: + #content only checked for the full run + assert np.min(output.ucts_timestamp) == np.uint64( + self.RUNS["expected_ucts_timestamp_min"][i] + ) + assert np.max(output.ucts_timestamp) == np.uint64( + self.RUNS["expected_ucts_timestamp_max"][i] + ) + assert output.charges_lg.min() == self.RUNS["charges_lg_min"][i] + assert output.charges_lg.max() == self.RUNS["charges_lg_max"][i] + assert int(10*output.charges_lg.mean()) == self.RUNS["charges_lg_mean"][i] + assert int(10*output.charges_lg.std()) ==self.RUNS["charges_lg_std"][i] + assert output.charges_hg.min() == self.RUNS["charges_hg_min"][i] + assert output.charges_hg.max() == self.RUNS["charges_hg_max"][i] + assert int(10*output.charges_hg.mean()) == self.RUNS["charges_hg_mean"][i] + assert int(10*output.charges_hg.std()) ==self.RUNS["charges_hg_std"][i] + + # Check output on disk + assert Path(outfile).exists() + + chargesContainers = ChargesContainers.from_hdf5(outfile) + ncontainers = 0 + for container in chargesContainers : + ncontainers +=1 + assert isinstance(container, ChargesContainers) + output = container.containers[self.RUNS["eventType"]] + if _events_per_slice is None : + expected_nevents = nevents + else : + if nevents%_events_per_slice == 0 : + expected_nevents = _events_per_slice + else : + if ncontainers == 1 : + expected_nevents = nevents%_events_per_slice + else : + expected_nevents = _events_per_slice + self.general_structure_testing( + output, + expected_nevents, + n_pixels, + run_number + ) + assert ncontainers == 1 if _events_per_slice is None else round(nevents / _events_per_slice) + + From 9e9ee9e85c0dc228fa5d3d71d07581735f9a00b4 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 19:10:47 +0100 Subject: [PATCH 19/36] formatting --- pyproject.toml | 10 + src/nectarchain/data/__init__.py | 35 +- src/nectarchain/data/container/__init__.py | 28 +- .../data/container/chargesContainer.py | 22 +- src/nectarchain/data/container/core.py | 109 +++--- src/nectarchain/data/container/eventSource.py | 20 +- .../data/container/gainContainer.py | 24 +- .../data/container/pedestalContainer.py | 8 +- .../data/container/tests/test_pedestal.py | 57 +-- .../data/container/waveformsContainer.py | 25 +- src/nectarchain/data/management.py | 25 +- src/nectarchain/display/__init__.py | 7 +- src/nectarchain/display/display.py | 9 +- src/nectarchain/dqm/charge_integration.py | 16 +- src/nectarchain/makers/__init__.py | 20 +- .../makers/calibration/pedestalMakers.py | 106 +++--- .../calibration/tests/test_pedestal_tool.py | 55 +-- src/nectarchain/makers/chargesMakers.py | 37 +- src/nectarchain/makers/component/__init__.py | 26 +- src/nectarchain/makers/component/core.py | 17 +- .../component/photostatistic_algorithm.py | 33 +- .../component/photostatistic_component.py | 20 +- .../makers/component/spe/__init__.py | 17 +- .../makers/component/spe/parameters.py | 12 +- src/nectarchain/makers/core.py | 12 +- .../makers/tests/test_charges_makers.py | 203 ++++++----- src/nectarchain/makers/tests/test_core.py | 336 ++++++++++++------ .../makers/tests/test_waveforms_makers.py | 181 ++++++---- src/nectarchain/makers/waveformsMakers.py | 10 +- .../ggrolleron/gain_PhotoStat_computation.py | 20 +- .../gain_SPEfit_combined_computation.py | 14 +- .../ggrolleron/gain_SPEfit_computation.py | 14 +- .../ggrolleron/load_wfs_compute_charge.py | 8 +- 33 files changed, 930 insertions(+), 606 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7e98a0d4..36e93a1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,3 +70,13 @@ exclude = ["nectarchain._dev_version"] [tool.setuptools_scm] write_to = "src/nectarchain/_version.py" + +[tool.ruff.lint] +select = ["E", "F", "W"] # This ensures it mimics Flake8's error codes. +exclude = ["build/", "dist/", "*.pyc"] + +[tool.ruff] +line-length = 88 + +[tool.black] +line-length = 88 \ No newline at end of file diff --git a/src/nectarchain/data/__init__.py b/src/nectarchain/data/__init__.py index 1506a100..c89b3978 100644 --- a/src/nectarchain/data/__init__.py +++ b/src/nectarchain/data/__init__.py @@ -1,2 +1,33 @@ -from .container import * -from .management import * +"""Description: This file is used to import all the classes and functions from the data module.""" + +from .container import ( + ArrayDataContainer, + ChargesContainer, + ChargesContainers, + GainContainer, + NectarCAMContainer, + NectarCAMPedestalContainer, + SPEfitContainer, + TriggerMapContainer, + WaveformsContainer, + WaveformsContainers, + get_array_keys, + merge_map_ArrayDataContainer, +) +from .management import DataManagement + +__all__ = [ + "ArrayDataContainer", + "NectarCAMContainer", + "TriggerMapContainer", + "get_array_keys", + "merge_map_ArrayDataContainer", + "ChargesContainer", + "ChargesContainers", + "WaveformsContainer", + "WaveformsContainers", + "GainContainer", + "SPEfitContainer", + "NectarCAMPedestalContainer", + "DataManagement", +] diff --git a/src/nectarchain/data/container/__init__.py b/src/nectarchain/data/container/__init__.py index 5b525634..3a969d84 100644 --- a/src/nectarchain/data/container/__init__.py +++ b/src/nectarchain/data/container/__init__.py @@ -1,4 +1,7 @@ -from .chargesContainer import * +"""This file is used to import all the containerclasses in the data/container +folder.""" + +from .chargesContainer import ChargesContainer, ChargesContainers from .core import ( ArrayDataContainer, NectarCAMContainer, @@ -6,7 +9,22 @@ get_array_keys, merge_map_ArrayDataContainer, ) -from .eventSource import * -from .gainContainer import * -from .waveformsContainer import * -from .pedestalContainer import * +from .gainContainer import GainContainer, SPEfitContainer +from .pedestalContainer import NectarCAMPedestalContainer, PedestalFlagBits +from .waveformsContainer import WaveformsContainer, WaveformsContainers + +__all__ = [ + "ArrayDataContainer", + "NectarCAMContainer", + "TriggerMapContainer", + "get_array_keys", + "merge_map_ArrayDataContainer", + "ChargesContainer", + "ChargesContainers", + "WaveformsContainer", + "WaveformsContainers", + "GainContainer", + "SPEfitContainer", + "NectarCAMPedestalContainer", + "PedestalFlagBits", +] diff --git a/src/nectarchain/data/container/chargesContainer.py b/src/nectarchain/data/container/chargesContainer.py index 02ef5603..23a3bd5a 100644 --- a/src/nectarchain/data/container/chargesContainer.py +++ b/src/nectarchain/data/container/chargesContainer.py @@ -1,20 +1,20 @@ import logging -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import numpy as np from ctapipe.containers import Field, Map, partial from .core import ArrayDataContainer, TriggerMapContainer +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["ChargesContainer", "ChargesContainers"] class ChargesContainer(ArrayDataContainer): - """ - A container that holds information about charges from a specific run. + """A container that holds information about charges from a specific run. Fields: charges_hg (np.ndarray): An array of high gain charges. @@ -40,13 +40,13 @@ class ChargesContainer(ArrayDataContainer): class ChargesContainers(TriggerMapContainer): - """ - Class representing a ChargesContainers. - - This class inherits from the `TriggerMapContainer` class and is used to store trigger or slices of data mappings of `ChargesContainer`. + """Class representing a ChargesContainers.This class inherits from the + `TriggerMapContainer` class and is used to store trigger or slices of data + mappings of `ChargesContainer`. Attributes: - containers (Field): A field representing the trigger or slices of data mapping of `ChargesContainer`. + containers (Field): A field representing the trigger or slices + of data mapping of `ChargesContainer`. """ containers = Field( diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index 90a120bf..ab645c6c 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -1,11 +1,6 @@ -import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import copy import importlib +import logging from pathlib import Path import numpy as np @@ -14,6 +9,11 @@ from ctapipe.io import HDF5TableReader from tables.exceptions import NoSuchNodeError +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = [ "ArrayDataContainer", "TriggerMapContainer", @@ -23,8 +23,8 @@ def get_array_keys(container: Container): - """ - Return a list of keys corresponding to fields which are array type in the given container. + """Return a list of keys corresponding to fields which are array type in the given + container. Parameters: container (Container): The container object to search for array fields. @@ -48,15 +48,14 @@ def get_array_keys(container: Container): class NectarCAMContainer(Container): - """ - Base class for the NectarCAM containers. This container cannot be recursive, - to be directly written with a HDF5TableWriter. + """Base class for the NectarCAM containers. + + This container cannot be recursive, to be directly written with a HDF5TableWriter. """ @staticmethod def _container_from_hdf5(path, container_class, index_component=0): - """ - Static method to read a container from an HDF5 file. + """Static method to read a container from an HDF5 file. Parameters: path (str or Path): The path to the HDF5 file. @@ -66,7 +65,8 @@ def _container_from_hdf5(path, container_class, index_component=0): Container: The container from the data in the HDF5 file. Example: - >>> container = NectarCAMContainer._container_from_hdf5('path_to_file.h5', MyContainerClass) + >>> container = NectarCAMContainer._container_from_hdf5('path_to_file.h5', + MyContainerClass) """ if isinstance(path, str): path = Path(path) @@ -83,8 +83,7 @@ def _container_from_hdf5(path, container_class, index_component=0): @classmethod def from_hdf5(cls, path, index_component=0): - """ - Reads a container from an HDF5 file. + """Reads a container from an HDF5 file. Parameters: path (str or Path): The path to the HDF5 file. @@ -105,8 +104,7 @@ def from_hdf5(cls, path, index_component=0): class ArrayDataContainer(NectarCAMContainer): - """ - A container that holds information about waveforms from a specific run. + """A container that holds information about waveforms from a specific run. Attributes: run_number (int): The run number associated with the waveforms. @@ -174,17 +172,18 @@ class ArrayDataContainer(NectarCAMContainer): class TriggerMapContainer(Container): - """ - Class representing a TriggerMapContainer. + """Class representing a TriggerMapContainer. - This class inherits from the `Container` class and is used to store trigger mappings of containers. + This class inherits from the `Container` class and is used to store trigger mappings + of containers. Attributes: containers (Field): A field representing the trigger mapping of containers. Methods: is_empty(): Checks if the TriggerMapContainer is empty. - validate(): Validates the TriggerMapContainer by checking if all the containers mapped are filled by correct type. + validate(): Validates the TriggerMapContainer by checking if all the containers + mapped are filled by correct type. Example: >>> container = TriggerMapContainer() @@ -201,14 +200,13 @@ class TriggerMapContainer(Container): @classmethod def from_hdf5(cls, path, slice_index=None, index_component=0): - """ - Reads a container from an HDF5 file. + """Reads a container from an HDF5 file. Parameters: path (str or Path): The path to the HDF5 file. - slice_index (int, optional): The index of the slice of data within the hdf5 file to read. Default is None. - - This method will call the _container_from_hdf5 method with the container argument associated to its own class (ArrayDataContainer) + slice_index (int, optional): The index of the slice of data within the hdf5 file + to read. Default is None.This method will call the _container_from_hdf5 method + with the container argument associated to its own class (ArrayDataContainer) Yields: Container: The container generator linked to the HDF5 file. @@ -228,20 +226,23 @@ def from_hdf5(cls, path, slice_index=None, index_component=0): def _container_from_hdf5( path, container_class, slice_index=None, index_component=0 ): - """ - Reads a container from an HDF5 file. + """Reads a container from an HDF5 file. Parameters: path (str or Path): The path to the HDF5 file. container_class (Container): The class of the container to be read. - slice_index (int, optional): The index of the slice of data within the hdf5 file to read. Default is None. + slice_index (int, optional): The index of the slice of data within the hdf5 file + to read. Default is None. - This method first checks if the path is a string and converts it to a Path object if it is. - It then imports the module of the container class and creates an instance of the container class. + This method first checks if the path is a string and converts it to a Path object + if it is. + It then imports the module of the container class and creates an instance of the + container class. If the HDF5 file contains more than one slice and no slice index is provided, it reads all slices and yields a generator of containers. - If a slice index is provided, it reads only the specified slice and returns the container instance. + If a slice index is provided, it reads only the specified slice and returns the + container instance. Yields: Container: The container associated to the HDF5 file. @@ -251,7 +252,8 @@ def _container_from_hdf5( Exception: If any other error occurs. Example: - >>> container = ArrayDataContainer._container_from_hdf5('path_to_file.h5', MyContainerClass) + >>> container = ArrayDataContainer._container_from_hdf5('path_to_file.h5', + MyContainerClass) """ if isinstance(path, str): path = Path(path) @@ -261,7 +263,9 @@ def _container_from_hdf5( with HDF5TableReader(path) as reader: if len(reader._h5file.root.__members__) > 1 and slice_index is None: log.info( - f"reading {container_class.__name__} containing {len(reader._h5file.root.__members__)} slices, will return a generator" + f"reading {container_class.__name__} containing\ + {len(reader._h5file.root.__members__)}\ + slices, will return a generator" ) for data in reader._h5file.root.__members__: # container.containers[data] = eval(f"module.{container_class.__name__}s")() @@ -282,12 +286,15 @@ def _container_from_hdf5( table_name=f"/{data}/{_waveforms_data[0]}/{trigger.name}", containers=_container, ) - # container.containers[data].containers[trigger] = next(tableReader) + # container.containers[data].containers[trigger] = + # next(tableReader) container.containers[trigger] = next(tableReader) else: log.info( - f"there is {len(_waveforms_data)} entry corresponding to a {container_class} table save, unable to load" + f"there is {len(_waveforms_data)} entry corresponding \ + to a {container_class} table save,\ + unable to load" ) except NoSuchNodeError as err: log.warning(err) @@ -298,12 +305,15 @@ def _container_from_hdf5( else: if slice_index is None: log.info( - f"reading {container_class.__name__} containing a single slice, will return the {container_class.__name__} instance" + f"reading {container_class.__name__} containing\ + a single slice,\ + will return the {container_class.__name__} instance" ) data = "data" else: log.info( - f"reading slice {slice_index} of {container_class.__name__}, will return the {container_class.__name__} instance" + f"reading slice {slice_index} of {container_class.__name__},\ + will return the {container_class.__name__} instance" ) data = f"data_{slice_index}" for key, trigger in EventType.__members__.items(): @@ -324,7 +334,7 @@ def _container_from_hdf5( yield container def is_empty(self): - """This method check if the container is empty + """This method check if the container is empty. Returns: bool: True if the container is empty, False otherwise. @@ -332,7 +342,8 @@ def is_empty(self): return len(self.containers.keys()) == 0 def validate(self): - """apply the validate method recursively to all the containers that are mapped within the TriggerMapContainer + """Apply the validate method recursively to all the containers that are mapped + within the TriggerMapContainer. Raises: FieldValidationError: if one container is not valid. @@ -344,18 +355,21 @@ def validate(self): else: if not (isinstance(container, container_type)): raise FieldValidationError( - "all the containers mapped must have the same type to be merged " + "all the containers mapped must have the same type to be merged" ) def merge_map_ArrayDataContainer(triggerMapContainer: TriggerMapContainer): - """ - Merge and map ArrayDataContainer + """Merge and map ArrayDataContainer. - This function takes a TriggerMapContainer as input and merges the array fields of the containers mapped within the TriggerMapContainer. The merged array fields are concatenated along the 0th axis. The function also updates the 'nevents' field of the output container by summing the 'nevents' field of all the mapped containers. + This function takes a TriggerMapContainer as input and merges the array fields of + the containers mapped within the TriggerMapContainer. The merged array fields are + concatenated along the 0th axis. The function also updates the 'nevents' field of + the output container by summing the 'nevents' field of all the mapped containers. Parameters: - triggerMapContainer (TriggerMapContainer): The TriggerMapContainer object containing the containers to be merged and mapped. + triggerMapContainer (TriggerMapContainer): The TriggerMapContainer object + containing the containers to be merged and mapped. Returns: ArrayDataContainer: The merged and mapped ArrayDataContainer object. @@ -384,7 +398,8 @@ def merge_map_ArrayDataContainer(triggerMapContainer: TriggerMapContainer): """ triggerMapContainer.validate() log.warning( - "TAKE CARE TO MERGE CONTAINERS ONLY IF PIXELS ID, RUN_NUMBER (OR ANY FIELD THAT ARE NOT ARRAY) ARE THE SAME" + "TAKE CARE TO MERGE CONTAINERS ONLY IF PIXELS ID, RUN_NUMBER (OR ANY FIELD THAT\ + ARE NOT ARRAY) ARE THE SAME" ) keys = list(triggerMapContainer.containers.keys()) output_container = copy.deepcopy(triggerMapContainer.containers[keys[0]]) diff --git a/src/nectarchain/data/container/eventSource.py b/src/nectarchain/data/container/eventSource.py index bf55c8e5..ab8ce688 100644 --- a/src/nectarchain/data/container/eventSource.py +++ b/src/nectarchain/data/container/eventSource.py @@ -26,8 +26,7 @@ def fill_nectarcam_event_container_from_zfile(self, array_event, event): - """ - Fill the NectarCAM event container from the zfile event data. + """Fill the NectarCAM event container from the zfile event data. Parameters: - array_event: The NectarCAMDataContainer object to fill with event data. @@ -54,7 +53,6 @@ def fill_nectarcam_event_container_from_zfile(self, array_event, event): in the event_container. 7. Calls the unpack_feb_data function to unpack the FEB counters and trigger pattern from the event and assign them to the corresponding fields in the event_container. - """ tel_id = self.tel_id @@ -83,7 +81,7 @@ def fill_nectarcam_event_container_from_zfile(self, array_event, event): def unpack_feb_data(self, event_container, event): - """Unpack FEB counters and trigger pattern""" + """Unpack FEB counters and trigger pattern.""" # Deduce data format version bytes_per_module = ( len(event.nectarcam.counters) // self.nectarcam_service.num_modules @@ -117,8 +115,7 @@ def unpack_feb_data(self, event_container, event): def fill_trigger_info(self, array_event): - """ - Fill the trigger information for a given event. + """Fill the trigger information for a given event. Parameters: array_event (NectarCAMEventContainer): The NectarCAMEventContainer object to @@ -188,9 +185,8 @@ def fill_trigger_info(self, array_event): class LightNectarCAMEventSource(NectarCAMEventSource): - """ - LightNectarCAMEventSource is a subclass of NectarCAMEventSource that provides a - generator for iterating over NectarCAM events. + """LightNectarCAMEventSource is a subclass of NectarCAMEventSource that + provides a generator for iterating over NectarCAM events. This implementation of the NectarCAMEventSource is much lighter than the one within ctapipe_io_nectarcam, only the fields interesting for nectarchain are kept. @@ -218,9 +214,8 @@ class LightNectarCAMEventSource(NectarCAMEventSource): """ def _generator(self): - """ - The generator function that yields NectarCAMDataContainer objects representing - each event. + """The generator function that yields NectarCAMDataContainer objects + representing each event. Yields ------ @@ -230,7 +225,6 @@ def _generator(self): Raises ------ None - """ # container for NectarCAM data array_event = NectarCAMDataContainer() diff --git a/src/nectarchain/data/container/gainContainer.py b/src/nectarchain/data/container/gainContainer.py index d1950cf3..7cbffbf8 100644 --- a/src/nectarchain/data/container/gainContainer.py +++ b/src/nectarchain/data/container/gainContainer.py @@ -1,25 +1,27 @@ import logging -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import numpy as np from ctapipe.containers import Field from .core import NectarCAMContainer +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["GainContainer", "SPEfitContainer"] class GainContainer(NectarCAMContainer): - """ - Class representing a GainContainer. + """Class representing a GainContainer. - This class is a subclass of NectarCAMContainer and provides additional fields and methods specific to gain calibration data. + This class is a subclass of NectarCAMContainer and provides additional fields and + methods specific to gain calibration data. Attributes: - is_valid (np.ndarray): Array of booleans indicating the validity of each gain value. + is_valid (np.ndarray): Array of booleans indicating the validity of each gain + value. high_gain (np.ndarray): Array of high gain values. low_gain (np.ndarray): Array of low gain values. pixels_id (np.ndarray): Array of pixel IDs. @@ -54,10 +56,10 @@ def from_hdf5(cls, path): class SPEfitContainer(GainContainer): - """ - Class representing a SPEfitContainer. + """Class representing a SPEfitContainer. - This class is a subclass of GainContainer and provides additional fields specific to single photoelectron (SPE) fit data. + This class is a subclass of GainContainer and provides additional fields specific to + single photoelectron (SPE) fit data. Attributes: likelihood (np.ndarray): Array of likelihood values. diff --git a/src/nectarchain/data/container/pedestalContainer.py b/src/nectarchain/data/container/pedestalContainer.py index 77959b95..65add6be 100644 --- a/src/nectarchain/data/container/pedestalContainer.py +++ b/src/nectarchain/data/container/pedestalContainer.py @@ -10,11 +10,12 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers +__all__ = ["NectarCAMPedestalContainer", "PedestalFlagBits"] + @unique class PedestalFlagBits(IntFlag): - """ - Define the bits corresponding to pedestal-related flags + """Define the bits corresponding to pedestal-related flags. NEVENTS: Number of events below the acceptable minimum value MEAN_PEDESTAL: Mean pedestal outside acceptable range @@ -31,8 +32,7 @@ class PedestalFlagBits(IntFlag): class NectarCAMPedestalContainer(NectarCAMContainer): - """ - A container that holds estimated pedestals + """A container that holds estimated pedestals. Fields: nsamples (int): The number of samples in the waveforms. diff --git a/src/nectarchain/data/container/tests/test_pedestal.py b/src/nectarchain/data/container/tests/test_pedestal.py index e2c5e144..48fa3b7c 100644 --- a/src/nectarchain/data/container/tests/test_pedestal.py +++ b/src/nectarchain/data/container/tests/test_pedestal.py @@ -1,8 +1,11 @@ -import numpy as np import tempfile + +import numpy as np from ctapipe.io import HDF5TableWriter + from nectarchain.data.container import NectarCAMPedestalContainer + def generate_mock_pedestal_container(): # fixed values nchannels = 2 @@ -19,7 +22,7 @@ def generate_mock_pedestal_container(): pedestal_mean_lg = np.float64(np.random.uniform(240, 260, size=(npixels, nsamples))) pedestal_std_hg = np.float64(np.random.normal(size=(npixels, nsamples))) pedestal_std_lg = np.float64(np.random.normal(size=(npixels, nsamples))) - pixel_mask = np.int8(np.random.randint(0,2,size=(nchannels,npixels))) + pixel_mask = np.int8(np.random.randint(0, 2, size=(nchannels, npixels))) # create pedestal container pedestal_container = NectarCAMPedestalContainer( @@ -32,42 +35,47 @@ def generate_mock_pedestal_container(): pedestal_mean_lg=pedestal_mean_lg, pedestal_std_hg=pedestal_std_hg, pedestal_std_lg=pedestal_std_lg, - pixel_mask = pixel_mask + pixel_mask=pixel_mask, ) pedestal_container.validate() # create dictionary that duplicates content - dict = {'nsamples': nsamples, - 'nevents': nevents, - 'pixels_id': pixels_id, - 'ucts_timestamp_min': ucts_timestamp_min, - 'ucts_timestamp_max': ucts_timestamp_max, - 'pedestal_mean_hg': pedestal_mean_hg, - 'pedestal_mean_lg': pedestal_mean_lg, - 'pedestal_std_hg': pedestal_std_hg, - 'pedestal_std_lg': pedestal_std_lg, - 'pixel_mask': pixel_mask - } + dict = { + "nsamples": nsamples, + "nevents": nevents, + "pixels_id": pixels_id, + "ucts_timestamp_min": ucts_timestamp_min, + "ucts_timestamp_max": ucts_timestamp_max, + "pedestal_mean_hg": pedestal_mean_hg, + "pedestal_mean_lg": pedestal_mean_lg, + "pedestal_std_hg": pedestal_std_hg, + "pedestal_std_lg": pedestal_std_lg, + "pixel_mask": pixel_mask, + } # return both container and input content return pedestal_container, dict -class TestNectarCAMPedestalContainer: +class TestNectarCAMPedestalContainer: def test_create_pedestal_container_mem(self): # create mock pedestal container pedestal_container, dict = generate_mock_pedestal_container() # check that all fields are filled correctly with input values - assert pedestal_container.nsamples == dict['nsamples'] - assert pedestal_container.nevents.tolist() == dict['nevents'].tolist() - assert pedestal_container.ucts_timestamp_min == dict['ucts_timestamp_min'] - assert pedestal_container.ucts_timestamp_max == dict['ucts_timestamp_max'] - assert np.allclose(pedestal_container.pedestal_mean_hg, dict['pedestal_mean_hg']) - assert np.allclose(pedestal_container.pedestal_mean_lg, dict['pedestal_mean_lg']) - assert np.allclose(pedestal_container.pedestal_std_hg, dict['pedestal_std_hg']) - assert np.allclose(pedestal_container.pedestal_std_lg, dict['pedestal_std_lg']) - assert np.allclose(pedestal_container.pixel_mask, dict['pixel_mask']) + assert pedestal_container.nsamples == dict["nsamples"] + assert pedestal_container.nevents.tolist() == dict["nevents"].tolist() + assert pedestal_container.ucts_timestamp_min == dict["ucts_timestamp_min"] + assert pedestal_container.ucts_timestamp_max == dict["ucts_timestamp_max"] + assert np.allclose( + pedestal_container.pedestal_mean_hg, dict["pedestal_mean_hg"] + ) + assert np.allclose( + pedestal_container.pedestal_mean_lg, dict["pedestal_mean_lg"] + ) + assert np.allclose(pedestal_container.pedestal_std_hg, dict["pedestal_std_hg"]) + assert np.allclose(pedestal_container.pedestal_std_lg, dict["pedestal_std_lg"]) + assert np.allclose(pedestal_container.pixel_mask, dict["pixel_mask"]) # FIXME # Guillaume is working on generalizing the fromhdf5 method to all containers @@ -100,5 +108,6 @@ def test_create_pedestal_container_mem(self): # assert np.allclose(pedestal_container.pedestal_std_hg, dict['pedestal_std_hg']) # assert np.allclose(pedestal_container.pedestal_std_lg, dict['pedestal_std_lg']) + if __name__ == "__main__": pass diff --git a/src/nectarchain/data/container/waveformsContainer.py b/src/nectarchain/data/container/waveformsContainer.py index 05eb0fd7..f430b951 100644 --- a/src/nectarchain/data/container/waveformsContainer.py +++ b/src/nectarchain/data/container/waveformsContainer.py @@ -1,18 +1,19 @@ import logging -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import numpy as np from ctapipe.containers import Field, Map, partial from .core import ArrayDataContainer, TriggerMapContainer +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + +__all__ = ["WaveformsContainer", "WaveformsContainers"] + class WaveformsContainer(ArrayDataContainer): - """ - A container that holds information about waveforms from a specific run. + """A container that holds information about waveforms from a specific run. Fields: nsamples (int): The number of samples in the waveforms. @@ -25,7 +26,8 @@ class WaveformsContainer(ArrayDataContainer): type=np.uint8, description="number of samples in the waveforms", ) - # subarray = Field(type=SubarrayDescription, description="The subarray description") + # subarray = Field(type=SubarrayDescription, + # description="The subarray description") wfs_hg = Field( type=np.ndarray, dtype=np.uint16, ndim=3, description="high gain waveforms" ) @@ -35,13 +37,14 @@ class WaveformsContainer(ArrayDataContainer): class WaveformsContainers(TriggerMapContainer): - """ - Class representing a container for waveforms from specific runs. + """Class representing a container for waveforms from specific runs. - This class inherits from the `TriggerMapContainer` class and is used to store trigger or slices of data mappings of `WaveformsContainer` instances. + This class inherits from the `TriggerMapContainer` class and is used to store + trigger or slices of data mappings of `WaveformsContainer` instances. Attributes: - containers (Field): A field representing the trigger or slices of data mapping of `WaveformsContainer` instances. + containers (Field): A field representing the trigger or slices of data mapping + of `WaveformsContainer` instances. """ containers = Field( diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index 50914c51..6b970437 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -24,17 +24,15 @@ DIRAC.initialize() except ImportError: log.warning("DIRAC probably not installed") - pass except Exception as e: log.warning(f"DIRAC could not be properly initialized: {e}") - pass class DataManagement: @staticmethod def findrun(run_number: int, search_on_GRID=True) -> Tuple[Path, List[Path]]: - """method to find in NECTARCAMDATA the list of ``*.fits.fz`` files associated to - run_number + """Method to find in NECTARCAMDATA the list of ``*.fits.fz`` files + associated to run_number. Parameters ---------- @@ -45,7 +43,6 @@ def findrun(run_number: int, search_on_GRID=True) -> Tuple[Path, List[Path]]: ------- (PosixPath,list): the path list of ``*.fits.fz`` files - """ basepath = f"{os.environ['NECTARCAMDATA']}/runs/" list = glob.glob( @@ -82,7 +79,7 @@ def findrun(run_number: int, search_on_GRID=True) -> Tuple[Path, List[Path]]: @staticmethod def getRunFromDIRAC(lfns: list): - """Method to get run files from the EGI grid from input lfns + """Method to get run files from the EGI grid from input lfns. Parameters ---------- @@ -115,8 +112,7 @@ def get_GRID_location( username=None, password=None, ): - """ - Method to get run location on GRID from Elog (work in progress!) + """Method to get run location on GRID from Elog (work in progress!) Parameters ---------- @@ -139,7 +135,6 @@ def get_GRID_location( Returns ------- __get_GRID_location_ELog or __get_GRID_location_DIRAC - """ if fromElog: return __class__.__get_GRID_location_ELog( @@ -262,11 +257,13 @@ def __get_GRID_location_ELog( return lfns else: return url_data + @staticmethod def find_waveforms(run_number, max_events=None): return __class__.__find_computed_data( run_number=run_number, max_events=max_events, data_type="waveforms" ) + @staticmethod def find_charges( run_number, method="FullWaveformSum", str_extractor_kwargs="", max_events=None @@ -277,6 +274,7 @@ def find_charges( ext=f"_{method}_{str_extractor_kwargs}.h5", data_type="charges", ) + @staticmethod def find_photostat( FF_run_number, @@ -296,6 +294,7 @@ def find_photostat( if len(full_file) != 1: raise Exception(f"the files is {full_file}") return full_file + @staticmethod def find_SPE_combined( run_number, method="FullWaveformSum", str_extractor_kwargs="" @@ -306,6 +305,7 @@ def find_SPE_combined( str_extractor_kwargs=str_extractor_kwargs, keyword="FlatFieldCombined", ) + @staticmethod def find_SPE_nominal( run_number, method="FullWaveformSum", str_extractor_kwargs="", free_pp_n=False @@ -317,6 +317,7 @@ def find_SPE_nominal( free_pp_n=free_pp_n, keyword="FlatFieldSPENominal", ) + @staticmethod def find_SPE_HHV( run_number, @@ -334,8 +335,9 @@ def find_SPE_HHV( f"_{str_extractor_kwargs}.h5" ).__str__() ) - ###need to improve the files search !! - # -> unstable behavior with SPE results computed with maxevents not to None + # need to improve the files search !! + # -> unstable behavior with SPE results computed + # with maxevents not to None if len(full_file) != 1: all_files = glob.glob( pathlib.Path( @@ -357,6 +359,7 @@ def find_SPE_HHV( return [all_files[index]] else: return full_file + @staticmethod def __find_computed_data( run_number, max_events=None, ext=".h5", data_type="waveforms" diff --git a/src/nectarchain/display/__init__.py b/src/nectarchain/display/__init__.py index 4941e7d3..af897f07 100644 --- a/src/nectarchain/display/__init__.py +++ b/src/nectarchain/display/__init__.py @@ -1 +1,6 @@ -from .display import * +""" This module contains the display class for the container. +""" + +from .display import ContainerDisplay + +__all__ = ["ContainerDisplay"] diff --git a/src/nectarchain/display/display.py b/src/nectarchain/display/display.py index 73d57469..11c85737 100644 --- a/src/nectarchain/display/display.py +++ b/src/nectarchain/display/display.py @@ -1,6 +1,7 @@ import logging from abc import ABC +import numpy as np from ctapipe.visualization import CameraDisplay from matplotlib import pyplot as plt @@ -10,7 +11,7 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -import numpy as np +__all__ = ["ContainerDisplay"] class ContainerDisplay(ABC): @@ -33,10 +34,12 @@ def display(container: ArrayDataContainer, evt, geometry, cmap="gnuplot2"): pixels_id = container.pixels_id else: log.error( - "container can't be displayed, must be a ChargesContainer or a WaveformsContainer" + "container can't be displayed, must be a ChargesContainer or a\ + WaveformsContainer" ) raise Exception( - "container can't be displayed, must be a ChargesContainer or a WaveformsContainer" + "container can't be displayed, must be a ChargesContainer or a\ + WaveformsContainer" ) highlighten_pixels = np.array([], dtype=int) diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index 80f7517b..4e0aacae 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -1,13 +1,15 @@ import ctapipe.instrument.camera.readout import numpy as np from ctapipe.coordinates import EngineeringCameraFrame -from ctapipe.image.extractor import FixedWindowSum # noqa: F401 -from ctapipe.image.extractor import FullWaveformSum # noqa: F401 -from ctapipe.image.extractor import LocalPeakWindowSum # noqa: F401 -from ctapipe.image.extractor import NeighborPeakWindowSum # noqa: F401 -from ctapipe.image.extractor import SlidingWindowMaxSum # noqa: F401 -from ctapipe.image.extractor import TwoPassWindowSum # noqa: F401 -from ctapipe.image.extractor import GlobalPeakWindowSum +from ctapipe.image.extractor import ( + FixedWindowSum, + FullWaveformSum, + GlobalPeakWindowSum, + LocalPeakWindowSum, + NeighborPeakWindowSum, + SlidingWindowMaxSum, + TwoPassWindowSum, +) from ctapipe.visualization import CameraDisplay from ctapipe_io_nectarcam import constants from matplotlib import pyplot as plt diff --git a/src/nectarchain/makers/__init__.py b/src/nectarchain/makers/__init__.py index d690d0f6..7a467b33 100644 --- a/src/nectarchain/makers/__init__.py +++ b/src/nectarchain/makers/__init__.py @@ -1,4 +1,16 @@ -# from .chargesMakers import * -from .chargesMakers import * -from .core import * -from .waveformsMakers import * +""" Description: This file is used to import all the classes from the different files in the makers folder. +""" + +from .chargesMakers import ChargesNectarCAMCalibrationTool +from .core import ( + DelimiterLoopNectarCAMCalibrationTool, + EventsLoopNectarCAMCalibrationTool, +) +from .waveformsMakers import WaveformsNectarCAMCalibrationTool + +__all__ = [ + "ChargesNectarCAMCalibrationTool", + "DelimiterLoopNectarCAMCalibrationTool", + "EventsLoopNectarCAMCalibrationTool", + "WaveformsNectarCAMCalibrationTool", +] diff --git a/src/nectarchain/makers/calibration/pedestalMakers.py b/src/nectarchain/makers/calibration/pedestalMakers.py index a3f2c1ab..8519ce9b 100644 --- a/src/nectarchain/makers/calibration/pedestalMakers.py +++ b/src/nectarchain/makers/calibration/pedestalMakers.py @@ -8,15 +8,14 @@ log.handlers = logging.getLogger("__main__").handlers import pathlib -import tables +import tables from ctapipe.core.traits import ComponentNameList +from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN, N_GAINS -from ctapipe_io_nectarcam.constants import N_GAINS, HIGH_GAIN, LOW_GAIN - -from .core import NectarCAMCalibrationTool -from ..component import NectarCAMComponent from ...data.container import NectarCAMPedestalContainer +from ..component import NectarCAMComponent +from .core import NectarCAMCalibrationTool __all__ = ["PedestalNectarCAMCalibrationTool"] @@ -24,10 +23,11 @@ class PedestalNectarCAMCalibrationTool(NectarCAMCalibrationTool): name = "PedestalNectarCAMCalibrationTool" - componentsList = ComponentNameList(NectarCAMComponent, - default_value=["PedestalEstimationComponent"], - help="List of Component names to be applied, the order will be respected", ).tag( - config=True) + componentsList = ComponentNameList( + NectarCAMComponent, + default_value=["PedestalEstimationComponent"], + help="List of Component names to be applied, the order will be respected", + ).tag(config=True) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -44,10 +44,13 @@ def _init_output_path(self): if self.max_events is None: filename = f"{self.name}_run{self.run_number}{ext}" else: - filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}{ext}" + filename = ( + f"{self.name}_run{self.run_number}_maxevents{self.max_events}{ext}" + ) self.output_path = pathlib.Path( - f"{os.environ.get('NECTARCAMDATA', '/tmp')}/PedestalEstimation/{filename}") + f"{os.environ.get('NECTARCAMDATA', '/tmp')}/PedestalEstimation/{filename}" + ) def _combine_results(self): """ @@ -61,50 +64,61 @@ def _combine_results(self): # Loop over sliced results to fill the combined results self.log.info("Combine sliced results") for i, result in enumerate(h5file.root.__members__): - if result == 'data_combined': - log.error('Trying to combine results that already contain combined data') + if result == "data_combined": + log.error( + "Trying to combine results that already contain combined data" + ) table = h5file.root[result][NectarCAMPedestalContainer.__name__][0] if i == 0: # initialize fields for the combined results based on first slice - nsamples = table['nsamples'] - nevents = np.zeros(len(table['nevents'])) - pixels_id = table['pixels_id'] - ucts_timestamp_min = table['ucts_timestamp_min'] - ucts_timestamp_max = table['ucts_timestamp_max'] - pedestal_mean_hg = np.zeros(np.shape(table['pedestal_mean_hg'])) - pedestal_mean_lg = np.zeros(np.shape(table['pedestal_mean_lg'])) - pedestal_std_hg = np.zeros(np.shape(table['pedestal_std_hg'])) - pedestal_std_lg = np.zeros(np.shape(table['pedestal_std_lg'])) + nsamples = table["nsamples"] + nevents = np.zeros(len(table["nevents"])) + pixels_id = table["pixels_id"] + ucts_timestamp_min = table["ucts_timestamp_min"] + ucts_timestamp_max = table["ucts_timestamp_max"] + pedestal_mean_hg = np.zeros(np.shape(table["pedestal_mean_hg"])) + pedestal_mean_lg = np.zeros(np.shape(table["pedestal_mean_lg"])) + pedestal_std_hg = np.zeros(np.shape(table["pedestal_std_hg"])) + pedestal_std_lg = np.zeros(np.shape(table["pedestal_std_lg"])) else: # otherwise consider the overall time interval - ucts_timestamp_min = np.minimum(ucts_timestamp_min, - table['ucts_timestamp_min']) - ucts_timestamp_max = np.maximum(ucts_timestamp_max, - table['ucts_timestamp_max']) + ucts_timestamp_min = np.minimum( + ucts_timestamp_min, table["ucts_timestamp_min"] + ) + ucts_timestamp_max = np.maximum( + ucts_timestamp_max, table["ucts_timestamp_max"] + ) # for all slices # derive from pixel mask a mask that sets usable pixels # accept only pixels for which no flags were raised - usable_pixels = table['pixel_mask'] == 0 + usable_pixels = table["pixel_mask"] == 0 # use a pixel only if it has no flag on either channel - usable_pixels = np.logical_and(usable_pixels[0], - usable_pixels[1]) + usable_pixels = np.logical_and(usable_pixels[0], usable_pixels[1]) # cumulated number of events - nevents += table['nevents'] * usable_pixels + nevents += table["nevents"] * usable_pixels # add mean, std sum elements - pedestal_mean_hg += table['pedestal_mean_hg'] \ - * table['nevents'][:, np.newaxis] \ - * usable_pixels[:, np.newaxis] - pedestal_mean_lg += table['pedestal_mean_lg'] \ - * table['nevents'][:, np.newaxis] \ - * usable_pixels[:, np.newaxis] - pedestal_std_hg += table['pedestal_std_hg'] ** 2 \ - * table['nevents'][:, np.newaxis] \ - * usable_pixels[:, np.newaxis] - pedestal_std_lg += table['pedestal_std_lg'] ** 2 \ - * table['nevents'][:, np.newaxis] \ - * usable_pixels[:, np.newaxis] + pedestal_mean_hg += ( + table["pedestal_mean_hg"] + * table["nevents"][:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) + pedestal_mean_lg += ( + table["pedestal_mean_lg"] + * table["nevents"][:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) + pedestal_std_hg += ( + table["pedestal_std_hg"] ** 2 + * table["nevents"][:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) + pedestal_std_lg += ( + table["pedestal_std_lg"] ** 2 + * table["nevents"][:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) # calculate final values of mean and std pedestal_mean_hg /= nevents[:, np.newaxis] @@ -119,12 +133,12 @@ def _combine_results(self): # PedestalComponent ped_stats = {} array_shape = np.append([N_GAINS], np.shape(pedestal_mean_hg)) - for statistic in ['mean', 'std']: + for statistic in ["mean", "std"]: ped_stat = np.zeros(array_shape) - if statistic == 'mean': + if statistic == "mean": ped_stat[HIGH_GAIN] = pedestal_mean_hg ped_stat[LOW_GAIN] = pedestal_mean_lg - elif statistic == 'std': + elif statistic == "std": ped_stat[HIGH_GAIN] = pedestal_std_hg ped_stat[LOW_GAIN] = pedestal_std_lg # Store the result in the dictionary @@ -169,7 +183,7 @@ def finish(self, return_output_component=False, *args, **kwargs): output = self._combine_results() # add combined results to output # re-initialise writer to store combined results - self._init_writer(sliced=True, group_name='data_combined') + self._init_writer(sliced=True, group_name="data_combined") # add combined results to writer self._write_container(output) self.writer.close() diff --git a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py index baea50c7..ec9d9822 100644 --- a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py +++ b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py @@ -51,7 +51,7 @@ def test_base(self): pixel_mask_nevents_min=1, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -165,7 +165,7 @@ def test_timesel(self): pixel_mask_nevents_min=1, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -218,7 +218,7 @@ def test_WaveformsStdFilter(self): pixel_mask_nevents_min=1, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -273,7 +273,7 @@ def test_ChargeDistributionFilter(self): pixel_mask_nevents_min=1, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -321,7 +321,7 @@ def test_pixel_mask(self): filter_method=None, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -331,9 +331,11 @@ def test_pixel_mask(self): flag_bit = PedestalFlagBits.NEVENTS assert np.all(output.pixel_mask & flag_bit == flag_bit) # Check that other flags were not raised - flag_bits = [PedestalFlagBits.MEAN_PEDESTAL, - PedestalFlagBits.STD_SAMPLE, - PedestalFlagBits.STD_PIXEL] + flag_bits = [ + PedestalFlagBits.MEAN_PEDESTAL, + PedestalFlagBits.STD_SAMPLE, + PedestalFlagBits.STD_PIXEL, + ] for flag_bit in flag_bits: assert np.all(output.pixel_mask & flag_bit == 0) @@ -351,8 +353,8 @@ def test_pixel_mask(self): overwrite=True, filter_method=None, pixel_mask_nevents_min=1, - pixel_mask_mean_min=1000., - pixel_mask_mean_max=1100., + pixel_mask_mean_min=1000.0, + pixel_mask_mean_max=1100.0, ) tool.initialize() @@ -365,9 +367,11 @@ def test_pixel_mask(self): flag_bit = PedestalFlagBits.MEAN_PEDESTAL assert np.all(output.pixel_mask & flag_bit == flag_bit) # Check that other flags were not raised - flag_bits = [PedestalFlagBits.NEVENTS, - PedestalFlagBits.STD_SAMPLE, - PedestalFlagBits.STD_PIXEL] + flag_bits = [ + PedestalFlagBits.NEVENTS, + PedestalFlagBits.STD_SAMPLE, + PedestalFlagBits.STD_PIXEL, + ] for flag_bit in flag_bits: assert np.all(output.pixel_mask & flag_bit == 0) @@ -382,7 +386,7 @@ def test_pixel_mask(self): overwrite=True, filter_method=None, pixel_mask_nevents_min=1, - pixel_mask_std_sample_min=100. + pixel_mask_std_sample_min=100.0, ) tool.initialize() @@ -391,13 +395,16 @@ def test_pixel_mask(self): tool.start() output = tool.finish(return_output_component=True)[0] - # Check that all pixels were flagged as having a small sample std + # Check that all pixels were flagged as + # having a small sample std flag_bit = PedestalFlagBits.STD_SAMPLE assert np.all(output.pixel_mask & flag_bit == flag_bit) # Check that other flags were not raised - flag_bits = [PedestalFlagBits.NEVENTS, - PedestalFlagBits.MEAN_PEDESTAL, - PedestalFlagBits.STD_PIXEL] + flag_bits = [ + PedestalFlagBits.NEVENTS, + PedestalFlagBits.MEAN_PEDESTAL, + PedestalFlagBits.STD_PIXEL, + ] for flag_bit in flag_bits: assert np.all(output.pixel_mask & flag_bit == 0) @@ -412,10 +419,10 @@ def test_pixel_mask(self): overwrite=True, filter_method=None, pixel_mask_nevents_min=1, - pixel_mask_std_pixel_max=0.01 + pixel_mask_std_pixel_max=0.01, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -425,8 +432,10 @@ def test_pixel_mask(self): flag_bit = PedestalFlagBits.STD_PIXEL assert np.all(output.pixel_mask & flag_bit == flag_bit) # Check that other flags were not raised - flag_bits = [PedestalFlagBits.NEVENTS, - PedestalFlagBits.MEAN_PEDESTAL, - PedestalFlagBits.STD_SAMPLE] + flag_bits = [ + PedestalFlagBits.NEVENTS, + PedestalFlagBits.MEAN_PEDESTAL, + PedestalFlagBits.STD_SAMPLE, + ] for flag_bit in flag_bits: assert np.all(output.pixel_mask & flag_bit == 0) diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/chargesMakers.py index 7fad5e43..8662d9bc 100644 --- a/src/nectarchain/makers/chargesMakers.py +++ b/src/nectarchain/makers/chargesMakers.py @@ -1,14 +1,8 @@ import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import os import pathlib import numpy as np -from ctapipe.containers import EventType from ctapipe.core.traits import Bool, ComponentNameList from ctapipe.image.extractor import ( BaselineSubtractedNeighborPeakWindowSum, @@ -21,17 +15,17 @@ TwoPassWindowSum, ) -from ..data.container import ( - ChargesContainers, - TriggerMapContainer, - WaveformsContainer, - WaveformsContainers, -) +from ..data.container import WaveformsContainer, WaveformsContainers from ..data.management import DataManagement from .component import ChargesComponent, NectarCAMComponent from .core import EventsLoopNectarCAMCalibrationTool from .extractor.utils import CtapipeExtractor +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["ChargesNectarCAMCalibrationTool"] @@ -59,9 +53,11 @@ def _init_output_path(self): self.extractor_kwargs ) if self.max_events is None: - filename = f"{self.name}_run{self.run_number}_{self.method}_{str_extractor_kwargs}.h5" + filename = f"{self.name}_run{self.run_number}_{self.method}" + f"{str_extractor_kwargs}.h5" else: - filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}_{self.method}_{str_extractor_kwargs}.h5" + filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}_" + f"{self.method}_{str_extractor_kwargs}.h5" self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/runs/charges/{filename}" @@ -82,7 +78,9 @@ def start( ) if len(files) != 1: self.log.info( - f"{len(files)} computed wavforms files found with max_events >= {self.max_events} for run {self.run_number}, reload waveforms from event loop" + f"{len(files)} computed wavforms files found with max_events >=\ + {self.max_events} for run {self.run_number}, reload waveforms\ + from event loop" ) super().start( n_events=n_events, @@ -92,7 +90,8 @@ def start( ) else: self.log.info( - f"{files[0]} is the computed wavforms files found with max_events >= {self.max_events} for run {self.run_number}" + f"{files[0]} is the computed wavforms files found with max_events >=\ + {self.max_events} for run {self.run_number}" ) waveformsContainers = WaveformsContainers.from_hdf5(files[0]) if not (isinstance(waveformsContainers, WaveformsContainer)): @@ -117,7 +116,8 @@ def start( self._write_container(container=chargesContainers) else: self.log.debug( - f"WaveformsContainer file contains {n_slices} slices of the run events" + f"WaveformsContainer file contains {n_slices} slices of the\ + run events" ) for slice_index, _waveformsContainers in enumerate( waveformsContainers @@ -132,7 +132,8 @@ def start( self._write_container(container=chargesContainers) else: self.log.debug( - "WaveformsContainer file container is a simple WaveformsContainer (not mapped)" + "WaveformsContainer file container is a simple \ + WaveformsContainer (not mapped)" ) self._init_writer(sliced=False) chargesContainers = ChargesComponent.create_from_waveforms( diff --git a/src/nectarchain/makers/component/__init__.py b/src/nectarchain/makers/component/__init__.py index f7b85ec9..880c803d 100644 --- a/src/nectarchain/makers/component/__init__.py +++ b/src/nectarchain/makers/component/__init__.py @@ -1,12 +1,18 @@ -from .chargesComponent import * -from .core import * -from .FlatFieldSPEComponent import * -from .PedestalComponent import * -from .gainComponent import * -from .photostatistic_algorithm import * -from .photostatistic_component import * -from .spe import * -from .waveformsComponent import * +from .chargesComponent import ChargesComponent +from .core import ArrayDataComponent, NectarCAMComponent, get_valid_component +from .FlatFieldSPEComponent import ( + FlatFieldCombinedSPEStdNectarCAMComponent, + FlatFieldSingleHHVSPENectarCAMComponent, + FlatFieldSingleHHVSPEStdNectarCAMComponent, + FlatFieldSingleNominalSPENectarCAMComponent, + FlatFieldSingleNominalSPEStdNectarCAMComponent, +) +from .gainComponent import GainNectarCAMComponent +from .PedestalComponent import PedestalEstimationComponent +from .photostatistic_algorithm import PhotoStatisticAlgorithm +from .photostatistic_component import PhotoStatisticNectarCAMComponent +from .spe import SPECombinedalgorithm, SPEHHValgorithm, SPEHHVStdalgorithm +from .waveformsComponent import WaveformsComponent __all__ = [ "ArrayDataComponent", @@ -19,9 +25,11 @@ "FlatFieldSingleNominalSPENectarCAMComponent", "FlatFieldSingleNominalSPEStdNectarCAMComponent", "FlatFieldCombinedSPEStdNectarCAMComponent", + "get_valid_component", "ChargesComponent", "WaveformsComponent", "PedestalEstimationComponent", "PhotoStatisticNectarCAMComponent", "PhotoStatisticAlgorithm", + "GainNectarCAMComponent", ] diff --git a/src/nectarchain/makers/component/core.py b/src/nectarchain/makers/component/core.py index eb9f2072..1966a2a4 100644 --- a/src/nectarchain/makers/component/core.py +++ b/src/nectarchain/makers/component/core.py @@ -9,8 +9,8 @@ from ctapipe.core.traits import ComponentNameList, Integer, Unicode from ctapipe.instrument import CameraGeometry from ctapipe_io_nectarcam import constants -from ctapipe_io_nectarcam.containers import NectarCAMDataContainer from ctapipe_io_nectarcam.constants import N_PIXELS +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer from ...data.container.core import ArrayDataContainer @@ -226,13 +226,16 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.__ucts_event_counter[f"{name}"].append( event.nectarcam.tel[__class__.TEL_ID.default_value].evt.ucts_event_counter ) - if event.nectarcam.tel[__class__.TEL_ID.default_value].evt.trigger_pattern is None : - self.__trig_patter_all[f"{name}"].append( - np.empty((4,N_PIXELS)).T - ) - else : + if ( + event.nectarcam.tel[__class__.TEL_ID.default_value].evt.trigger_pattern + is None + ): + self.__trig_patter_all[f"{name}"].append(np.empty((4, N_PIXELS)).T) + else: self.__trig_patter_all[f"{name}"].append( - event.nectarcam.tel[__class__.TEL_ID.default_value].evt.trigger_pattern.T + event.nectarcam.tel[ + __class__.TEL_ID.default_value + ].evt.trigger_pattern.T ) if kwargs.get("return_wfs", False): diff --git a/src/nectarchain/makers/component/photostatistic_algorithm.py b/src/nectarchain/makers/component/photostatistic_algorithm.py index 21ba8e26..c9e45743 100644 --- a/src/nectarchain/makers/component/photostatistic_algorithm.py +++ b/src/nectarchain/makers/component/photostatistic_algorithm.py @@ -1,15 +1,7 @@ -import logging -import sys - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - - import copy +import logging import os -import matplotlib import numpy as np from astropy.visualization import quantity_support from ctapipe.core import Component @@ -19,6 +11,11 @@ from ...data.container import ChargesContainer, GainContainer, SPEfitContainer from ..component import ChargesComponent +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["PhotoStatisticAlgorithm"] @@ -119,7 +116,8 @@ def __get_charges_FF_Ped_reshaped( def __check_shape(self) -> None: """ - Checks the shape of certain attributes and raises an exception if the shape is not as expected. + Checks the shape of certain attributes and raises an exception if the shape is + not as expected. """ try: self.__FFcharge_hg[0] * self.__FFcharge_lg[0] * self.__Pedcharge_hg[ @@ -160,14 +158,16 @@ def plot_correlation( photoStat_gain: np.ndarray, SPE_gain: np.ndarray ) -> plt.Figure: """ - Plot the correlation between the photo statistic gain and the single photoelectron (SPE) gain. + Plot the correlation between the photo statistic gain and the single + photoelectron (SPE) gain. Args: photoStat_gain (np.ndarray): Array of photo statistic gain values. SPE_gain (np.ndarray): Array of SPE gain values. Returns: - fig (plt.Figure): The figure object containing the scatter plot and the linear fit line. + fig (plt.Figure): The figure object containing the scatter plot + and the linear fit line. """ # matplotlib.use("TkAgg") # Create a mask to filter the data points based on certain criteria @@ -197,7 +197,8 @@ def plot_correlation( x, y(x), color="red", - label=f"linear fit,\n a = {a:.2e},\n b = {b:.2e},\n r = {r:.2e},\n p_value = {p_value:.2e},\n std_err = {std_err:.2e}", + label=f"linear fit,\n a = {a:.2e},\n b = {b:.2e},\n r = {r:.2e},\n\ + p_value = {p_value:.2e},\n std_err = {std_err:.2e}", ) # Plot the line y = x @@ -222,7 +223,8 @@ def SPE_resolution(self) -> float: @property def sigmaPedHG(self) -> float: """ - Calculates and returns the standard deviation of Pedcharge_hg multiplied by the square root of coefCharge_FF_Ped. + Calculates and returns the standard deviation of Pedcharge_hg multiplied by the + square root of coefCharge_FF_Ped. Returns: float: The standard deviation of Pedcharge_hg. @@ -297,7 +299,8 @@ def gainHG(self) -> float: @property def sigmaPedLG(self) -> float: """ - Calculates and returns the standard deviation of Pedcharge_lg multiplied by the square root of coefCharge_FF_Ped. + Calculates and returns the standard deviation of Pedcharge_lg multiplied by the + square root of coefCharge_FF_Ped. Returns: float: The standard deviation of Pedcharge_lg. diff --git a/src/nectarchain/makers/component/photostatistic_component.py b/src/nectarchain/makers/component/photostatistic_component.py index 0b73e38d..1c5ea0ed 100644 --- a/src/nectarchain/makers/component/photostatistic_component.py +++ b/src/nectarchain/makers/component/photostatistic_component.py @@ -1,10 +1,5 @@ -import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import copy +import logging from ctapipe.containers import EventType from ctapipe.core.traits import List, Path, Unicode @@ -17,12 +12,18 @@ from .gainComponent import GainNectarCAMComponent from .photostatistic_algorithm import PhotoStatisticAlgorithm +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["PhotoStatisticNectarCAMComponent"] class PhotoStatisticNectarCAMComponent(GainNectarCAMComponent): SPE_result = Path( - help="the path of the SPE result container computed with very high voltage data", + help="the path of the SPE result container computed with very\ + high voltage data", ).tag(config=True) PhotoStatAlgorithm = Unicode( "PhotoStatisticAlgorithm", @@ -103,7 +104,8 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.Ped_chargesComponent(event=event, *args, **kwargs) else: self.log.warning( - f"event {event.index.event_id} is event type {event.trigger.event_type} which is not used here" + f"event {event.index.event_id} is event type {event.trigger.event_type}\ + which is not used here" ) def finish(self, *args, **kwargs): @@ -130,5 +132,5 @@ def finish(self, *args, **kwargs): parent=self, **self._PhotoStatAlgorithm_kwargs, ) - fit_output = photo_stat.run(pixels_id=self.asked_pixels_id, *args, **kwargs) + _ = photo_stat.run(pixels_id=self.asked_pixels_id, *args, **kwargs) return photo_stat.results diff --git a/src/nectarchain/makers/component/spe/__init__.py b/src/nectarchain/makers/component/spe/__init__.py index 4933f073..479a9e51 100644 --- a/src/nectarchain/makers/component/spe/__init__.py +++ b/src/nectarchain/makers/component/spe/__init__.py @@ -1,2 +1,15 @@ -# from .parameters import * -from .spe_algorithm import * +from .spe_algorithm import ( + SPECombinedalgorithm, + SPEHHValgorithm, + SPEHHVStdalgorithm, + SPEnominalalgorithm, + SPEnominalStdalgorithm, +) + +__all__ = [ + "SPEHHValgorithm", + "SPEHHVStdalgorithm", + "SPEnominalStdalgorithm", + "SPEnominalalgorithm", + "SPECombinedalgorithm", +] diff --git a/src/nectarchain/makers/component/spe/parameters.py b/src/nectarchain/makers/component/spe/parameters.py index 28d384a3..26db7aae 100644 --- a/src/nectarchain/makers/component/spe/parameters.py +++ b/src/nectarchain/makers/component/spe/parameters.py @@ -1,12 +1,12 @@ +import copy import logging +import astropy.units as u +import numpy as np + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -import copy - -import astropy.units as u -import numpy as np __all__ = ["Parameter", "Parameters"] @@ -42,7 +42,9 @@ def from_instance(cls, parameter): ) def __str__(self): - return f"name : {self.__name}, value : {self.__value}, error : {self.__error}, unit : {self.__unit}, min : {self.__min}, max : {self.__max},frozen : {self.__frozen}" + return f"name : {self.__name}, value : {self.__value}, error : {self.__error},\ + unit : {self.__unit}, min : {self.__min}, max : {self.__max},\ + frozen : {self.__frozen}" @property def name(self): diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 8c12bed8..af45c8cc 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -16,15 +16,15 @@ classes_with_traits, flag, ) -from ctapipe_io_nectarcam.containers import NectarCAMDataContainer from ctapipe.io import HDF5TableWriter from ctapipe.io.datawriter import DATA_MODEL_VERSION +from ctapipe_io_nectarcam import LightNectarCAMEventSource +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer from tables.exceptions import HDF5ExtError from tqdm.auto import tqdm from traitlets import default from ..data import DataManagement -from ctapipe_io_nectarcam import LightNectarCAMEventSource from ..data.container.core import NectarCAMContainer, TriggerMapContainer from ..utils import ComponentUtils from .component import NectarCAMComponent, get_valid_component @@ -434,7 +434,9 @@ def start( self._setup_components() n_events_in_slice = 0 - def split_run(self, n_events_in_slice : int = None, event : NectarCAMDataContainer = None): + def split_run( + self, n_events_in_slice: int = None, event: NectarCAMDataContainer = None + ): """Method to decide if criteria to end a run slice are met""" condition = ( self.events_per_slice is not None @@ -552,7 +554,9 @@ class DelimiterLoopNectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def split_run(self, n_events_in_slice : int = None, event : NectarCAMDataContainer = None): + def split_run( + self, n_events_in_slice: int = None, event: NectarCAMDataContainer = None + ): """Method to decide if criteria to end a run slice is met""" condition = event.trigger.event_type == EventType.UNKNOWN return condition diff --git a/src/nectarchain/makers/tests/test_charges_makers.py b/src/nectarchain/makers/tests/test_charges_makers.py index 5978fdf0..89deb3ba 100644 --- a/src/nectarchain/makers/tests/test_charges_makers.py +++ b/src/nectarchain/makers/tests/test_charges_makers.py @@ -1,47 +1,49 @@ -import logging import tempfile +from pathlib import Path + import numpy as np -import pytest from ctapipe.containers import EventType -from pathlib import Path +from ctapipe.utils import get_dataset_path + from nectarchain.data.container import ChargesContainer, ChargesContainers from nectarchain.makers import ChargesNectarCAMCalibrationTool -from ctapipe.utils import get_dataset_path "This test file test the overall workflow of the ChargesNectarCAMCalibrationTool, adapted from the test of the WaveformsNectarCAMCalibrationTool. There are lot of code duplication but I don't care for now. Do you ?" -class TestChargesNectarCAMCalibrationTool : +class TestChargesNectarCAMCalibrationTool: RUNS = { - "Run number": [3938, 5288], - "Run file": [ - get_dataset_path("NectarCAM.Run3938.30events.fits.fz"), - get_dataset_path("NectarCAM.Run5288.0001.fits.fz"), - ], - "nevents" : [30,13], - "N pixels": [1834, 1848], - "eventType" : EventType.SKY_PEDESTAL, - "charges_lg_min" : [957,973], - "charges_lg_max" : [2059,2113], - "charges_lg_mean" : [19067,19306], - "charges_lg_std" : [2273,2376], - "charges_hg_min" : [963,972], - "charges_hg_max" : [2057,2109], - "charges_hg_mean" : [19106,19275], - "charges_hg_std" : [2139,2357], - "expected_ucts_timestamp_min" : [1674462932637854793, 1715007113924900896], - "expected_ucts_timestamp_max" : [1674462932695877994, 1715007123524920096], -} + "Run number": [3938, 5288], + "Run file": [ + get_dataset_path("NectarCAM.Run3938.30events.fits.fz"), + get_dataset_path("NectarCAM.Run5288.0001.fits.fz"), + ], + "nevents": [30, 13], + "N pixels": [1834, 1848], + "eventType": EventType.SKY_PEDESTAL, + "charges_lg_min": [957, 973], + "charges_lg_max": [2059, 2113], + "charges_lg_mean": [19067, 19306], + "charges_lg_std": [2273, 2376], + "charges_hg_min": [963, 972], + "charges_hg_max": [2057, 2109], + "charges_hg_mean": [19106, 19275], + "charges_hg_std": [2139, 2357], + "expected_ucts_timestamp_min": [1674462932637854793, 1715007113924900896], + "expected_ucts_timestamp_max": [1674462932695877994, 1715007123524920096], + } OVERWRITE = True METHOD = "LocalPeakWindowSum" EXTRACTOR_KWARGS = {"window_shift": 4, "window_width": 8} - - def general_structure_testing(self,output : ChargesContainer,nevents:int,n_pixels:int,run_number:int) : + + def general_structure_testing( + self, output: ChargesContainer, nevents: int, n_pixels: int, run_number: int + ): assert isinstance(output.pixels_id, np.ndarray) assert output.pixels_id.dtype == np.uint16 assert np.shape(output.pixels_id) == (n_pixels,) assert output.run_number == run_number - assert output.camera == 'NectarCam-003' + assert output.camera == "NectarCam-003" assert output.npixels == n_pixels assert isinstance(output.ucts_busy_counter, np.ndarray) assert output.ucts_busy_counter.dtype == np.uint32 @@ -56,17 +58,17 @@ def general_structure_testing(self,output : ChargesContainer,nevents:int,n_pixel assert output.trig_pattern_all.dtype == bool assert isinstance(output.multiplicity, np.ndarray) assert output.multiplicity.dtype == np.uint16 - assert isinstance(output.ucts_timestamp,np.ndarray) + assert isinstance(output.ucts_timestamp, np.ndarray) assert output.ucts_timestamp.dtype == np.uint64 - - assert isinstance(output.event_id,np.ndarray) + + assert isinstance(output.event_id, np.ndarray) assert output.event_id.dtype == np.uint32 - assert isinstance(output.broken_pixels_hg,np.ndarray) + assert isinstance(output.broken_pixels_hg, np.ndarray) assert output.broken_pixels_hg.dtype == bool - assert output.broken_pixels_hg.shape == (nevents,n_pixels) - assert isinstance(output.broken_pixels_lg,np.ndarray) + assert output.broken_pixels_hg.shape == (nevents, n_pixels) + assert isinstance(output.broken_pixels_lg, np.ndarray) assert output.broken_pixels_lg.dtype == bool - assert output.broken_pixels_lg.shape == (nevents,n_pixels) + assert output.broken_pixels_lg.shape == (nevents, n_pixels) assert output.charges_hg.shape == (nevents, n_pixels) assert output.charges_lg.shape == (nevents, n_pixels) assert isinstance(output.charges_hg, np.ndarray) @@ -79,20 +81,18 @@ def general_structure_testing(self,output : ChargesContainer,nevents:int,n_pixel assert isinstance(output.peak_lg, np.ndarray) assert output.peak_hg.dtype == np.uint16 assert output.peak_lg.dtype == np.uint16 - assert isinstance(output.method,str) + assert isinstance(output.method, str) assert output.method == self.METHOD - def test_base(self): """ Test basic functionality, including IO on disk """ - events_per_slice =[None,None,10,11,8] - max_events = [None,10,None,None,10] + events_per_slice = [None, None, 10, 11, 8] + max_events = [None, 10, None, None, 10] - - for _max_events,_events_per_slice in zip(max_events,events_per_slice): + for _max_events, _events_per_slice in zip(max_events, events_per_slice): for i, run_number in enumerate(self.RUNS["Run number"]): run_file = self.RUNS["Run file"][i] n_pixels = self.RUNS["N pixels"][i] @@ -108,77 +108,112 @@ def test_base(self): log_level=0, output_path=outfile, overwrite=self.OVERWRITE, - method = self.METHOD, - extractor_kwargs = self.EXTRACTOR_KWARGS, + method=self.METHOD, + extractor_kwargs=self.EXTRACTOR_KWARGS, ) tool.setup() nevents = len(tool.event_source) - assert nevents == self.RUNS["nevents"][i] if _max_events is None else _max_events + assert ( + nevents == self.RUNS["nevents"][i] + if _max_events is None + else _max_events + ) tool.start() output_containers = tool.finish(return_output_component=True)[0] assert isinstance(output_containers, ChargesContainers) output = output_containers.containers[self.RUNS["eventType"]] assert isinstance(output, ChargesContainer) # Check output in memory - if _events_per_slice is not None and nevents%_events_per_slice == 0 : + if ( + _events_per_slice is not None + and nevents % _events_per_slice == 0 + ): assert output.nevents is None - else : - if _events_per_slice is None : - assert output.nevents == nevents #nevents has been validated before - else : - assert output.nevents == nevents%_events_per_slice - + else: + if _events_per_slice is None: + assert ( + output.nevents == nevents + ) # nevents has been validated before + else: + assert output.nevents == nevents % _events_per_slice + self.general_structure_testing( output, - nevents if _events_per_slice is None else nevents%_events_per_slice, + ( + nevents + if _events_per_slice is None + else nevents % _events_per_slice + ), n_pixels, - run_number - ) - - + run_number, + ) + if _events_per_slice is None and _max_events is None: - #content only checked for the full run + # content only checked for the full run assert np.min(output.ucts_timestamp) == np.uint64( - self.RUNS["expected_ucts_timestamp_min"][i] + self.RUNS["expected_ucts_timestamp_min"][i] ) assert np.max(output.ucts_timestamp) == np.uint64( self.RUNS["expected_ucts_timestamp_max"][i] ) - assert output.charges_lg.min() == self.RUNS["charges_lg_min"][i] - assert output.charges_lg.max() == self.RUNS["charges_lg_max"][i] - assert int(10*output.charges_lg.mean()) == self.RUNS["charges_lg_mean"][i] - assert int(10*output.charges_lg.std()) ==self.RUNS["charges_lg_std"][i] - assert output.charges_hg.min() == self.RUNS["charges_hg_min"][i] - assert output.charges_hg.max() == self.RUNS["charges_hg_max"][i] - assert int(10*output.charges_hg.mean()) == self.RUNS["charges_hg_mean"][i] - assert int(10*output.charges_hg.std()) ==self.RUNS["charges_hg_std"][i] + assert ( + output.charges_lg.min() + == self.RUNS["charges_lg_min"][i] + ) + assert ( + output.charges_lg.max() + == self.RUNS["charges_lg_max"][i] + ) + assert ( + int(10 * output.charges_lg.mean()) + == self.RUNS["charges_lg_mean"][i] + ) + assert ( + int(10 * output.charges_lg.std()) + == self.RUNS["charges_lg_std"][i] + ) + assert ( + output.charges_hg.min() + == self.RUNS["charges_hg_min"][i] + ) + assert ( + output.charges_hg.max() + == self.RUNS["charges_hg_max"][i] + ) + assert ( + int(10 * output.charges_hg.mean()) + == self.RUNS["charges_hg_mean"][i] + ) + assert ( + int(10 * output.charges_hg.std()) + == self.RUNS["charges_hg_std"][i] + ) # Check output on disk assert Path(outfile).exists() - - chargesContainers = ChargesContainers.from_hdf5(outfile) + + chargesContainers = ChargesContainers.from_hdf5(outfile) ncontainers = 0 - for container in chargesContainers : - ncontainers +=1 + for container in chargesContainers: + ncontainers += 1 assert isinstance(container, ChargesContainers) output = container.containers[self.RUNS["eventType"]] - if _events_per_slice is None : + if _events_per_slice is None: expected_nevents = nevents - else : - if nevents%_events_per_slice == 0 : + else: + if nevents % _events_per_slice == 0: expected_nevents = _events_per_slice - else : - if ncontainers == 1 : - expected_nevents = nevents%_events_per_slice - else : + else: + if ncontainers == 1: + expected_nevents = nevents % _events_per_slice + else: expected_nevents = _events_per_slice self.general_structure_testing( - output, - expected_nevents, - n_pixels, - run_number - ) - assert ncontainers == 1 if _events_per_slice is None else round(nevents / _events_per_slice) - - + output, expected_nevents, n_pixels, run_number + ) + assert ( + ncontainers == 1 + if _events_per_slice is None + else round(nevents / _events_per_slice) + ) diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core.py index 602e3c6f..ed9c049d 100644 --- a/src/nectarchain/makers/tests/test_core.py +++ b/src/nectarchain/makers/tests/test_core.py @@ -1,29 +1,24 @@ import logging -from ctapipe.utils import get_dataset_path -from nectarchain.makers.core import ( - BaseNectarCAMCalibrationTool, - EventsLoopNectarCAMCalibrationTool, - DelimiterLoopNectarCAMCalibrationTool - ) -from ctapipe.containers import EventType -from nectarchain.data import NectarCAMContainer -from nectarchain.makers.component import NectarCAMComponent -from ctapipe_io_nectarcam.containers import NectarCAMDataContainer -from ctapipe_io_nectarcam import LightNectarCAMEventSource import os import pathlib -import pytest -from unittest.mock import patch +from unittest.mock import MagicMock, patch + import numpy as np +import pytest import traitlets +from ctapipe.containers import EventType, TriggerContainer from ctapipe.core.container import Container -from nectarchain.data.container.core import ( - NectarCAMContainer, - TriggerMapContainer, -) -from ctapipe.containers import TriggerContainer -from unittest.mock import MagicMock +from ctapipe.utils import get_dataset_path +from ctapipe_io_nectarcam import LightNectarCAMEventSource +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer +from nectarchain.data.container.core import NectarCAMContainer, TriggerMapContainer +from nectarchain.makers.component import NectarCAMComponent +from nectarchain.makers.core import ( + BaseNectarCAMCalibrationTool, + DelimiterLoopNectarCAMCalibrationTool, + EventsLoopNectarCAMCalibrationTool, +) logging.basicConfig( format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG @@ -35,39 +30,45 @@ class TestBaseNectarCAMCalibrationTool: RUN_NUMBER = 3938 RUN_FILE = get_dataset_path("NectarCAM.Run3938.30events.fits.fz") - + def test_load_run(self): - eventsource = BaseNectarCAMCalibrationTool.load_run(run_number = self.RUN_NUMBER,max_events=1,run_file = self.RUN_FILE) - assert isinstance(eventsource,LightNectarCAMEventSource) + eventsource = BaseNectarCAMCalibrationTool.load_run( + run_number=self.RUN_NUMBER, max_events=1, run_file=self.RUN_FILE + ) + assert isinstance(eventsource, LightNectarCAMEventSource) + class MockComponent(NectarCAMComponent): def __init__(self, *args, **kwargs): pass + def __call__(self, event, *args, **kwargs): pass + def start(self): pass + def finish(self): return [NectarCAMContainer()] + class TestEventsLoopNectarCAMCalibrationTool(TestBaseNectarCAMCalibrationTool): MAX_EVENTS = 10 EVENTS_PER_SLICE = 8 - - @pytest.fixture def tool_instance(self): return EventsLoopNectarCAMCalibrationTool( run_number=self.RUN_NUMBER, - ) + ) + @pytest.fixture def tool_instance_run_file(self): return EventsLoopNectarCAMCalibrationTool( run_number=self.RUN_NUMBER, run_file=self.RUN_FILE, - ) - + ) + def test_init_output_path(self, tool_instance): expected_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA', '/tmp')}/runs/EventsLoopNectarCAMCalibration_run{self.RUN_NUMBER}.h5" @@ -78,70 +79,92 @@ def test_init_output_path(self, tool_instance): assert tool_instance.run_file is None assert tool_instance.name == "EventsLoopNectarCAMCalibration" assert tool_instance.events_per_slice is None - + def test_init_with_output_path(self): custom_path = pathlib.Path("/custom/path/output.h5") - tool_instance = EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER, output_path=custom_path) + tool_instance = EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, output_path=custom_path + ) assert tool_instance.output_path == custom_path - + def test_init_with_max_events(self): - tool_instance = EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER, max_events = 10) + tool_instance = EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, max_events=10 + ) assert tool_instance.max_events == self.MAX_EVENTS - + def test_init_with_events_per_slice(self): - tool_instance = EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER, events_per_slice = 4) + tool_instance = EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, events_per_slice=self.EVENTS_PER_SLICE + ) assert tool_instance.events_per_slice == self.EVENTS_PER_SLICE - + def test_init_with_run_file(self): - tool_instance = EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER, run_file = self.RUN_FILE) + tool_instance = EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, run_file=self.RUN_FILE + ) assert tool_instance.run_file == self.RUN_FILE def test_load_eventsource(self, tool_instance_run_file): tool_instance_run_file._load_eventsource() - assert isinstance(tool_instance_run_file.event_source, LightNectarCAMEventSource) + assert isinstance( + tool_instance_run_file.event_source, LightNectarCAMEventSource + ) assert tool_instance_run_file.event_source.input_url == self.RUN_FILE + def test_load_eventsource_max_events(self, tool_instance_run_file): tool_instance_run_file.max_events = self.MAX_EVENTS tool_instance_run_file._load_eventsource() - assert isinstance(tool_instance_run_file.event_source, LightNectarCAMEventSource) + assert isinstance( + tool_instance_run_file.event_source, LightNectarCAMEventSource + ) assert tool_instance_run_file.event_source.input_url == self.RUN_FILE assert tool_instance_run_file.event_source.max_events == self.MAX_EVENTS @patch("nectarchain.makers.core.HDF5TableWriter") @patch("nectarchain.makers.core.os.remove") @patch("nectarchain.makers.core.os.makedirs") - def test_init_writer_full_mode(self, mock_makedirs, mock_remove, mock_writer, tool_instance): + def test_init_writer_full_mode( + self, mock_makedirs, mock_remove, mock_writer, tool_instance + ): tool_instance.overwrite = True tool_instance._init_writer(sliced=False) mock_remove.assert_called_once_with(tool_instance.output_path) - mock_makedirs.assert_called_once_with(tool_instance.output_path.parent, exist_ok=True) + mock_makedirs.assert_called_once_with( + tool_instance.output_path.parent, exist_ok=True + ) mock_writer.assert_called_once_with( filename=tool_instance.output_path, parent=tool_instance, mode="w", group_name="data", ) - - - + @patch("nectarchain.makers.core.HDF5TableWriter") @patch("nectarchain.makers.core.os.remove") @patch("nectarchain.makers.core.os.makedirs") - def test_init_writer_sliced_mode(self, mock_makedirs, mock_remove, mock_writer, tool_instance): + def test_init_writer_sliced_mode( + self, mock_makedirs, mock_remove, mock_writer, tool_instance + ): tool_instance.overwrite = True tool_instance._init_writer(sliced=True, slice_index=1) mock_remove.assert_not_called() - mock_makedirs.assert_called_once_with(tool_instance.output_path.parent, exist_ok=True) + mock_makedirs.assert_called_once_with( + tool_instance.output_path.parent, exist_ok=True + ) mock_writer.assert_called_once_with( filename=tool_instance.output_path, parent=tool_instance, mode="a", group_name="data_1", ) + @patch("nectarchain.makers.core.HDF5TableWriter") @patch("nectarchain.makers.core.os.remove") @patch("nectarchain.makers.core.os.makedirs") - def test_init_writer_overwrite_false(self, mock_makedirs, mock_remove, mock_writer, tool_instance): + def test_init_writer_overwrite_false( + self, mock_makedirs, mock_remove, mock_writer, tool_instance + ): tool_instance.overwrite = False with patch("nectarchain.makers.core.os.path.exists", return_value=True): with pytest.raises(Exception): @@ -149,28 +172,60 @@ def test_init_writer_overwrite_false(self, mock_makedirs, mock_remove, mock_writ mock_remove.assert_not_called() mock_makedirs.assert_not_called() mock_writer.assert_not_called() - + def test_setup_eventsource(self, tool_instance_run_file): tool_instance_run_file._setup_eventsource() - - assert tool_instance_run_file._npixels == tool_instance_run_file.event_source.nectarcam_service.num_pixels - assert np.all(tool_instance_run_file._pixels_id == tool_instance_run_file.event_source.nectarcam_service.pixel_ids) - assert isinstance(tool_instance_run_file.event_source,LightNectarCAMEventSource) - - - @patch("nectarchain.makers.core.ComponentUtils.get_class_name_from_ComponentName", return_value="ValidComponentClass") - @patch("nectarchain.makers.core.ComponentUtils.get_configurable_traits", return_value={'trait1': 'value1'}) - def test_get_provided_component_kwargs(self,mock_get_class_name,mock_get_valid_component,tool_instance): + + assert ( + tool_instance_run_file._npixels + == tool_instance_run_file.event_source.nectarcam_service.num_pixels + ) + assert np.all( + tool_instance_run_file._pixels_id + == tool_instance_run_file.event_source.nectarcam_service.pixel_ids + ) + assert isinstance( + tool_instance_run_file.event_source, LightNectarCAMEventSource + ) + + @patch( + "nectarchain.makers.core.ComponentUtils.get_class_name_from_ComponentName", + return_value="ValidComponentClass", + ) + @patch( + "nectarchain.makers.core.ComponentUtils.get_configurable_traits", + return_value={"trait1": "value1"}, + ) + def test_get_provided_component_kwargs( + self, mock_get_class_name, mock_get_valid_component, tool_instance + ): tool_instance.trait1 = "value1" - output_component_kwargs = tool_instance._get_provided_component_kwargs('componentName') + output_component_kwargs = tool_instance._get_provided_component_kwargs( + "componentName" + ) assert output_component_kwargs == {"trait1": "value1"} - - + @patch("nectarchain.makers.core.Component") - @patch("nectarchain.makers.core.get_valid_component", return_value=["WaveformsComponent"]) - @patch("nectarchain.makers.core.ComponentUtils.get_class_name_from_ComponentName", return_value="WaveformsComponentClass") - @patch("nectarchain.makers.core.ComponentUtils.get_configurable_traits", return_value={"trait1": "value1"}) - def test_setup_components(self, mock_get_configurable_traits, mock_get_class_name, mock_get_valid_component, mock_component, tool_instance_run_file): + @patch( + "nectarchain.makers.core.get_valid_component", + return_value=["WaveformsComponent"], + ) + @patch( + "nectarchain.makers.core.ComponentUtils.get_class_name_from_ComponentName", + return_value="WaveformsComponentClass", + ) + @patch( + "nectarchain.makers.core.ComponentUtils.get_configurable_traits", + return_value={"trait1": "value1"}, + ) + def test_setup_components( + self, + mock_get_configurable_traits, + mock_get_class_name, + mock_get_valid_component, + mock_component, + tool_instance_run_file, + ): with pytest.raises(traitlets.traitlets.TraitError): tool_instance_run_file.componentsList = ["ValidComponent"] tool_instance_run_file.componentsList = ["WaveformsComponent"] @@ -184,109 +239,145 @@ def test_setup_components(self, mock_get_configurable_traits, mock_get_class_nam "WaveformsComponent", subarray=tool_instance_run_file.event_source.subarray, parent=tool_instance_run_file, - trait1="value1" + trait1="value1", ) assert len(tool_instance_run_file.components) == 1 - assert tool_instance_run_file.components[0] == mock_component.from_name.return_value - - + assert ( + tool_instance_run_file.components[0] + == mock_component.from_name.return_value + ) + @patch("nectarchain.makers.core.os.remove") - @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_eventsource") - @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_components") + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_eventsource" + ) + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_components" + ) @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._init_writer") - def test_setup(self, mock_init_writer, mock_setup_components, mock_setup_eventsource, mock_remove, tool_instance_run_file): + def test_setup( + self, + mock_init_writer, + mock_setup_components, + mock_setup_eventsource, + mock_remove, + tool_instance_run_file, + ): tool_instance_run_file.overwrite = True tool_instance_run_file.output_path = pathlib.Path("/tmp/test_output.h5") - + with patch("nectarchain.makers.core.pathlib.Path.exists", return_value=True): tool_instance_run_file.setup() - + mock_setup_eventsource.assert_called_once() mock_setup_components.assert_called_once() mock_remove.assert_called_once_with(tool_instance_run_file.output_path) mock_init_writer.assert_called_once_with(sliced=False, slice_index=1) assert tool_instance_run_file._n_traited_events == 0 - def test_setup_run_number_not_set(self, tool_instance): tool_instance.run_number = -1 with pytest.raises(Exception, match="run_number need to be set up"): tool_instance.setup() - - def test_split_run(self,tool_instance): + + def test_split_run(self, tool_instance): event = NectarCAMDataContainer() - assert not(tool_instance.split_run(n_events_in_slice = 6,event=event)) + assert not (tool_instance.split_run(n_events_in_slice=6, event=event)) tool_instance.events_per_slice = 4 - assert tool_instance.split_run(n_events_in_slice = 6,event = event) - assert not(tool_instance.split_run(n_events_in_slice =2,event = event)) + assert tool_instance.split_run(n_events_in_slice=6, event=event) + assert not (tool_instance.split_run(n_events_in_slice=2, event=event)) @patch("nectarchain.makers.core.Component") - def test_start(self,mock_component,tool_instance_run_file): + def test_start(self, mock_component, tool_instance_run_file): tool_instance_run_file.overwrite = True tool_instance_run_file.setup() n_events = len(tool_instance_run_file.event_source) tool_instance_run_file.components = [mock_component.from_name.return_value] tool_instance_run_file.start() assert tool_instance_run_file._n_traited_events == n_events - + @patch("nectarchain.makers.core.Component") - def test_start_n_events(self,mock_component,tool_instance_run_file): + def test_start_n_events(self, mock_component, tool_instance_run_file): tool_instance_run_file.overwrite = True tool_instance_run_file.setup() tool_instance_run_file.components = [mock_component.from_name.return_value] - tool_instance_run_file.start(n_events = 10) + tool_instance_run_file.start(n_events=10) assert tool_instance_run_file._n_traited_events == 10 - + @patch("nectarchain.makers.core.Component") - @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components") - @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_components") - def test_start_sliced(self,mock_setup_components, mock_finish_components,mock_component,tool_instance_run_file): + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_components" + ) + def test_start_sliced( + self, + mock_setup_components, + mock_finish_components, + mock_component, + tool_instance_run_file, + ): tool_instance_run_file.overwrite = True tool_instance_run_file.events_per_slice = self.EVENTS_PER_SLICE tool_instance_run_file.setup() n_events = len(tool_instance_run_file.event_source) tool_instance_run_file.components = [MockComponent()] tool_instance_run_file.start() - assert mock_finish_components.call_count == n_events//self.EVENTS_PER_SLICE - assert mock_setup_components.call_count == n_events//self.EVENTS_PER_SLICE + 1 - + assert mock_finish_components.call_count == n_events // self.EVENTS_PER_SLICE + assert mock_setup_components.call_count == n_events // self.EVENTS_PER_SLICE + 1 + @patch("nectarchain.makers.core.Component") - @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components") - def test_finish(self, mock_finish_components, mock_component, tool_instance_run_file): + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) + def test_finish( + self, mock_finish_components, mock_component, tool_instance_run_file + ): tool_instance_run_file.overwrite = True tool_instance_run_file.setup() tool_instance_run_file.components = [mock_component.from_name.return_value] mock_finish_components.return_value = ["output"] - + output = tool_instance_run_file.finish() - + mock_finish_components.assert_called_once() assert output is None assert tool_instance_run_file.writer.h5file.isopen == 0 + @patch("nectarchain.makers.core.Component") - @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components") - def test_finish_with_output(self, mock_finish_components, mock_component, tool_instance_run_file): + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) + def test_finish_with_output( + self, mock_finish_components, mock_component, tool_instance_run_file + ): tool_instance_run_file.overwrite = True tool_instance_run_file.setup() tool_instance_run_file.components = [mock_component.from_name.return_value] - + output = tool_instance_run_file.finish(return_output_component=True) - + mock_finish_components.assert_called_once() assert output is not None assert tool_instance_run_file.writer.h5file.isopen == 0 - @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components") + + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) def test_finish_components(self, mock_finish_components, tool_instance_run_file): tool_instance_run_file.overwrite = True tool_instance_run_file.setup() tool_instance_run_file.components = [MockComponent()] - + output = tool_instance_run_file._finish_components() - + assert mock_finish_components.called_with([MockComponent().finish()], 0) @patch("nectarchain.makers.core.HDF5TableWriter") - def test_write_container_with_nectarcam_container(self, mock_writer, tool_instance_run_file): + def test_write_container_with_nectarcam_container( + self, mock_writer, tool_instance_run_file + ): tool_instance_run_file.writer = mock_writer container = MagicMock(spec=NectarCAMContainer) container.validate = MagicMock() @@ -296,9 +387,11 @@ def test_write_container_with_nectarcam_container(self, mock_writer, tool_instan table_name=f"{container.__class__.__name__}_0", containers=container, ) - + @patch("nectarchain.makers.core.HDF5TableWriter") - def test_write_container_with_triggermap_container(self, mock_writer, tool_instance_run_file): + def test_write_container_with_triggermap_container( + self, mock_writer, tool_instance_run_file + ): tool_instance_run_file.writer = mock_writer container = MagicMock(spec=TriggerMapContainer) container.validate = MagicMock() @@ -316,38 +409,45 @@ def test_write_container_with_triggermap_container(self, mock_writer, tool_insta table_name=f"{container.containers[EventType.UNKNOWN].__class__.__name__}_0/{EventType.UNKNOWN.name}", containers=container.containers[EventType.UNKNOWN], ) - + @patch("nectarchain.makers.core.HDF5TableWriter") - def test_write_container_with_invalid_container(self, mock_writer, tool_instance_run_file): + def test_write_container_with_invalid_container( + self, mock_writer, tool_instance_run_file + ): tool_instance_run_file.writer = mock_writer container = MagicMock(spec=Container) container.validate = MagicMock() - with pytest.raises(TypeError, match="component output must be an instance of TriggerMapContainer or NectarCAMContainer"): + with pytest.raises( + TypeError, + match="component output must be an instance of TriggerMapContainer or NectarCAMContainer", + ): tool_instance_run_file._write_container(container, index_component=0) container.validate.assert_called_once() mock_writer.write.assert_not_called() - - + @patch("nectarchain.makers.core.HDF5TableWriter") - def test_write_container_with_generic_exception(self, mock_writer, tool_instance_run_file): + def test_write_container_with_generic_exception( + self, mock_writer, tool_instance_run_file + ): tool_instance_run_file.writer = mock_writer container = MagicMock(spec=NectarCAMContainer) container.validate = MagicMock(side_effect=Exception("Generic error")) - with patch.object(tool_instance_run_file.log, 'error') as mock_log_error: + with patch.object(tool_instance_run_file.log, "error") as mock_log_error: with pytest.raises(Exception, match="Generic error"): tool_instance_run_file._write_container(container, index_component=0) container.validate.assert_called_once() mock_writer.write.assert_not_called() mock_log_error.assert_called_with("Generic error", exc_info=True) - - + class TestDelimiterLoopNectarCAMCalibrationTool: - def test_split_run(self) : + def test_split_run(self): tool = DelimiterLoopNectarCAMCalibrationTool() - event = NectarCAMDataContainer(trigger = TriggerContainer(event_type = EventType.FLATFIELD)) - assert not(tool.split_run(event = event)) - event = NectarCAMDataContainer(trigger = TriggerContainer(event_type = EventType.UNKNOWN)) - assert tool.split_run(event = event) - - + event = NectarCAMDataContainer( + trigger=TriggerContainer(event_type=EventType.FLATFIELD) + ) + assert not (tool.split_run(event=event)) + event = NectarCAMDataContainer( + trigger=TriggerContainer(event_type=EventType.UNKNOWN) + ) + assert tool.split_run(event=event) diff --git a/src/nectarchain/makers/tests/test_waveforms_makers.py b/src/nectarchain/makers/tests/test_waveforms_makers.py index 18a14439..69fdc375 100644 --- a/src/nectarchain/makers/tests/test_waveforms_makers.py +++ b/src/nectarchain/makers/tests/test_waveforms_makers.py @@ -1,46 +1,48 @@ -import logging import tempfile +from pathlib import Path + import numpy as np -import pytest from ctapipe.containers import EventType -from pathlib import Path -from nectarchain.data.container import WaveformsContainer, WaveformsContainers -from nectarchain.makers import WaveformsNectarCAMCalibrationTool from ctapipe.utils import get_dataset_path from ctapipe_io_nectarcam.constants import N_SAMPLES +from nectarchain.data.container import WaveformsContainer, WaveformsContainers +from nectarchain.makers import WaveformsNectarCAMCalibrationTool + "This test file test the overall workflow of the WaveformsNectarCAMCalibrationTool, covering also the EventsLoopNectarCAMCalibrationTool, which cannot be tested on data because the ArrayDataComponent within the WaveformsNectarCAMCalibrationTool is an abstract Component. However the EventsLoopNectarCAMCalibrationTool is still covered by unit test using pytest patches to mock the Component behavior." -class TestWaveformsNectarCAMCalibrationTool : +class TestWaveformsNectarCAMCalibrationTool: RUNS = { - "Run number": [3938, 5288], - "Run file": [ - get_dataset_path("NectarCAM.Run3938.30events.fits.fz"), - get_dataset_path("NectarCAM.Run5288.0001.fits.fz"), - ], - "nevents" : [30,13], - "N pixels": [1834, 1848], - "eventType" : EventType.SKY_PEDESTAL, - "wfs_lg_min" : [230,233], - "wfs_lg_max" : [264,269], - "wfs_lg_mean" : [2468,2509], - "wfs_lg_std" : [33,36], - "wfs_hg_min" : [178,195], - "wfs_hg_max" : [288,298], - "wfs_hg_mean" : [2467,2508], - "wfs_hg_std" : [36,38], - "expected_ucts_timestamp_min" : [1674462932637854793, 1715007113924900896], - "expected_ucts_timestamp_max" : [1674462932695877994, 1715007123524920096], -} + "Run number": [3938, 5288], + "Run file": [ + get_dataset_path("NectarCAM.Run3938.30events.fits.fz"), + get_dataset_path("NectarCAM.Run5288.0001.fits.fz"), + ], + "nevents": [30, 13], + "N pixels": [1834, 1848], + "eventType": EventType.SKY_PEDESTAL, + "wfs_lg_min": [230, 233], + "wfs_lg_max": [264, 269], + "wfs_lg_mean": [2468, 2509], + "wfs_lg_std": [33, 36], + "wfs_hg_min": [178, 195], + "wfs_hg_max": [288, 298], + "wfs_hg_mean": [2467, 2508], + "wfs_hg_std": [36, 38], + "expected_ucts_timestamp_min": [1674462932637854793, 1715007113924900896], + "expected_ucts_timestamp_max": [1674462932695877994, 1715007123524920096], + } OVERWRITE = True - - def general_structure_testing(self,output : WaveformsContainer,nevents:int,n_pixels:int,run_number:int) : + + def general_structure_testing( + self, output: WaveformsContainer, nevents: int, n_pixels: int, run_number: int + ): assert isinstance(output.pixels_id, np.ndarray) assert output.pixels_id.dtype == np.uint16 assert np.shape(output.pixels_id) == (n_pixels,) assert output.run_number == run_number - assert output.camera == 'NectarCam-003' + assert output.camera == "NectarCam-003" assert output.npixels == n_pixels assert isinstance(output.ucts_busy_counter, np.ndarray) assert output.ucts_busy_counter.dtype == np.uint32 @@ -55,17 +57,17 @@ def general_structure_testing(self,output : WaveformsContainer,nevents:int,n_pix assert output.trig_pattern_all.dtype == bool assert isinstance(output.multiplicity, np.ndarray) assert output.multiplicity.dtype == np.uint16 - assert isinstance(output.ucts_timestamp,np.ndarray) + assert isinstance(output.ucts_timestamp, np.ndarray) assert output.ucts_timestamp.dtype == np.uint64 - - assert isinstance(output.event_id,np.ndarray) + + assert isinstance(output.event_id, np.ndarray) assert output.event_id.dtype == np.uint32 - assert isinstance(output.broken_pixels_hg,np.ndarray) + assert isinstance(output.broken_pixels_hg, np.ndarray) assert output.broken_pixels_hg.dtype == bool - assert output.broken_pixels_hg.shape == (nevents,n_pixels) - assert isinstance(output.broken_pixels_lg,np.ndarray) + assert output.broken_pixels_hg.shape == (nevents, n_pixels) + assert isinstance(output.broken_pixels_lg, np.ndarray) assert output.broken_pixels_lg.dtype == bool - assert output.broken_pixels_lg.shape == (nevents,n_pixels) + assert output.broken_pixels_lg.shape == (nevents, n_pixels) assert output.wfs_hg.shape == (nevents, n_pixels, N_SAMPLES) assert output.wfs_lg.shape == (nevents, n_pixels, N_SAMPLES) assert isinstance(output.wfs_hg, np.ndarray) @@ -73,17 +75,15 @@ def general_structure_testing(self,output : WaveformsContainer,nevents:int,n_pix assert output.wfs_hg.dtype == np.uint16 assert output.wfs_lg.dtype == np.uint16 - def test_base(self): """ Test basic functionality, including IO on disk """ - events_per_slice =[None,None,10,11,8] - max_events = [None,10,None,None,10] - + events_per_slice = [None, None, 10, 11, 8] + max_events = [None, 10, None, None, 10] - for _max_events,_events_per_slice in zip(max_events,events_per_slice): + for _max_events, _events_per_slice in zip(max_events, events_per_slice): for i, run_number in enumerate(self.RUNS["Run number"]): run_file = self.RUNS["Run file"][i] n_pixels = self.RUNS["N pixels"][i] @@ -103,73 +103,96 @@ def test_base(self): tool.setup() nevents = len(tool.event_source) - assert nevents == self.RUNS["nevents"][i] if _max_events is None else _max_events + assert ( + nevents == self.RUNS["nevents"][i] + if _max_events is None + else _max_events + ) tool.start() output_containers = tool.finish(return_output_component=True)[0] assert isinstance(output_containers, WaveformsContainers) output = output_containers.containers[self.RUNS["eventType"]] assert isinstance(output, WaveformsContainer) # Check output in memory - if _events_per_slice is not None and nevents%_events_per_slice == 0 : + if ( + _events_per_slice is not None + and nevents % _events_per_slice == 0 + ): assert output.nevents is None - else : + else: assert output.nsamples == N_SAMPLES - if _events_per_slice is None : - assert output.nevents == nevents #nevents has been validated before - else : - assert output.nevents == nevents%_events_per_slice - + if _events_per_slice is None: + assert ( + output.nevents == nevents + ) # nevents has been validated before + else: + assert output.nevents == nevents % _events_per_slice + self.general_structure_testing( output, - nevents if _events_per_slice is None else nevents%_events_per_slice, + ( + nevents + if _events_per_slice is None + else nevents % _events_per_slice + ), n_pixels, - run_number - ) - - + run_number, + ) + if _events_per_slice is None and _max_events is None: - #content only checked for the full run + # content only checked for the full run assert np.min(output.ucts_timestamp) == np.uint64( - self.RUNS["expected_ucts_timestamp_min"][i] + self.RUNS["expected_ucts_timestamp_min"][i] ) assert np.max(output.ucts_timestamp) == np.uint64( self.RUNS["expected_ucts_timestamp_max"][i] ) assert output.wfs_lg.min() == self.RUNS["wfs_lg_min"][i] assert output.wfs_lg.max() == self.RUNS["wfs_lg_max"][i] - assert int(10*output.wfs_lg.mean()) == self.RUNS["wfs_lg_mean"][i] - assert int(10*output.wfs_lg.std()) ==self.RUNS["wfs_lg_std"][i] + assert ( + int(10 * output.wfs_lg.mean()) + == self.RUNS["wfs_lg_mean"][i] + ) + assert ( + int(10 * output.wfs_lg.std()) + == self.RUNS["wfs_lg_std"][i] + ) assert output.wfs_hg.min() == self.RUNS["wfs_hg_min"][i] assert output.wfs_hg.max() == self.RUNS["wfs_hg_max"][i] - assert int(10*output.wfs_hg.mean()) == self.RUNS["wfs_hg_mean"][i] - assert int(10*output.wfs_hg.std()) ==self.RUNS["wfs_hg_std"][i] + assert ( + int(10 * output.wfs_hg.mean()) + == self.RUNS["wfs_hg_mean"][i] + ) + assert ( + int(10 * output.wfs_hg.std()) + == self.RUNS["wfs_hg_std"][i] + ) # Check output on disk assert Path(outfile).exists() - - waveformsContainers = WaveformsContainers.from_hdf5(outfile) + + waveformsContainers = WaveformsContainers.from_hdf5(outfile) ncontainers = 0 - for container in waveformsContainers : - ncontainers +=1 + for container in waveformsContainers: + ncontainers += 1 assert isinstance(container, WaveformsContainers) output = container.containers[self.RUNS["eventType"]] - if _events_per_slice is None : + if _events_per_slice is None: expected_nevents = nevents - else : - if nevents%_events_per_slice == 0 : + else: + if nevents % _events_per_slice == 0: expected_nevents = _events_per_slice - else : - if ncontainers == 1 : - expected_nevents = nevents%_events_per_slice - else : + else: + if ncontainers == 1: + expected_nevents = nevents % _events_per_slice + else: expected_nevents = _events_per_slice self.general_structure_testing( - output, - expected_nevents, - n_pixels, - run_number - ) - - assert ncontainers == 1 if _events_per_slice is None else round(nevents / _events_per_slice) - - + output, expected_nevents, n_pixels, run_number + ) + + assert ( + ncontainers == 1 + if _events_per_slice is None + else round(nevents / _events_per_slice) + ) diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveformsMakers.py index fb0f94cb..96cc6597 100644 --- a/src/nectarchain/makers/waveformsMakers.py +++ b/src/nectarchain/makers/waveformsMakers.py @@ -1,9 +1,4 @@ import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import os import pathlib @@ -12,6 +7,11 @@ from .component import NectarCAMComponent from .core import EventsLoopNectarCAMCalibrationTool +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["WaveformsNectarCAMCalibrationTool"] diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py index e7877ecc..e7ef2864 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py @@ -1,11 +1,11 @@ +import argparse +import copy +import glob import json import logging import os import sys from pathlib import Path -import argparse -import copy -import glob from nectarchain.data.management import DataManagement from nectarchain.makers.calibration import PhotoStatisticNectarCAMCalibrationTool @@ -124,16 +124,16 @@ def main( str_extractor_kwargs = CtapipeExtractor.get_extractor_kwargs_str( args.extractor_kwargs ) - #path = DataManagement.find_SPE_HHV( + # path = DataManagement.find_SPE_HHV( # run_number=args.HHV_run_number, # method=args.method, # str_extractor_kwargs=str_extractor_kwargs, - #) - #path = DataManagement.find_SPE_nominal( + # ) + # path = DataManagement.find_SPE_nominal( # run_number=args.HHV_run_number, # method=args.method, # str_extractor_kwargs=str_extractor_kwargs, - #) + # ) path = DataManagement.find_SPE_nominal( run_number=args.HHV_run_number, method="GlobalPeakWindowSum", @@ -206,9 +206,9 @@ def main( kwargs.pop("figpath") kwargs.pop("HHV_run_number") - kwargs['FF_run_number'] = [3937] - kwargs['Ped_run_number'] = [3938] - kwargs['overwrite'] = True + kwargs["FF_run_number"] = [3937] + kwargs["Ped_run_number"] = [3938] + kwargs["overwrite"] = True args.HHV_run_number = 3936 log.info(f"arguments passed to main are : {kwargs}") diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py index ef09117f..e965c9fa 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py @@ -218,13 +218,13 @@ def main( kwargs.pop("display") kwargs.pop("HHV_run_number") - #args.HHV_run_number = 3942 - #kwargs['run_number'] = [3936] - #kwargs['overwrite'] = True - #kwargs['asked_pixels_id'] = [45,600,800] - #kwargs['multiproc'] = False - #args.display = True - #args.figpath = "/home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + # args.HHV_run_number = 3942 + # kwargs['run_number'] = [3936] + # kwargs['overwrite'] = True + # kwargs['asked_pixels_id'] = [45,600,800] + # kwargs['multiproc'] = False + # args.display = True + # args.figpath = "/home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" log.info(f"arguments passed to main are : {kwargs}") main(log=log, **kwargs) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py index 2ea069e6..4bcb6010 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py @@ -216,13 +216,13 @@ def main( kwargs.pop("HHV") kwargs.pop("free_pp_n") - #kwargs['run_number'] = [3942] - #kwargs['overwrite'] = True - #kwargs['asked_pixels_id'] = [45,600,800] - #args.HHV = True - #kwargs['multiproc'] = True - #args.display = True - #args.figpath = "/home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + # kwargs['run_number'] = [3942] + # kwargs['overwrite'] = True + # kwargs['asked_pixels_id'] = [45,600,800] + # args.HHV = True + # kwargs['multiproc'] = True + # args.display = True + # args.figpath = "/home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" log.info(f"arguments passed to main are : {kwargs}") main(log=log, **kwargs) diff --git a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py index 83c901db..d435cad2 100644 --- a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py +++ b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py @@ -210,8 +210,8 @@ def main( kwargs.pop("verbosity") log.info(f"arguments passed to main are : {kwargs}") - #kwargs['reload_wfs'] = True - #kwargs['run_number'] = [3784]#[5436] - #kwargs['overwrite'] = True - #kwargs['events_per_slice'] = 2000 + # kwargs['reload_wfs'] = True + # kwargs['run_number'] = [3784]#[5436] + # kwargs['overwrite'] = True + # kwargs['events_per_slice'] = 2000 main(log=log, **kwargs) From f11d0c60d5f8b56fdf3cc6681b31faa923a5388f Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 19:24:04 +0100 Subject: [PATCH 20/36] fix test core makers --- src/nectarchain/makers/__init__.py | 3 ++- src/nectarchain/makers/tests/test_core.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/nectarchain/makers/__init__.py b/src/nectarchain/makers/__init__.py index 7a467b33..1c55395a 100644 --- a/src/nectarchain/makers/__init__.py +++ b/src/nectarchain/makers/__init__.py @@ -1,4 +1,5 @@ -""" Description: This file is used to import all the classes from the different files in the makers folder. +""" Description: This file is used to import all the classes from the different files in +the makers folder. """ from .chargesMakers import ChargesNectarCAMCalibrationTool diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core.py index ed9c049d..91b5b3df 100644 --- a/src/nectarchain/makers/tests/test_core.py +++ b/src/nectarchain/makers/tests/test_core.py @@ -297,12 +297,18 @@ def test_start(self, mock_component, tool_instance_run_file): assert tool_instance_run_file._n_traited_events == n_events @patch("nectarchain.makers.core.Component") - def test_start_n_events(self, mock_component, tool_instance_run_file): + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) + def test_start_n_events(self, mock_finish_component, + mock_component, tool_instance_run_file): tool_instance_run_file.overwrite = True tool_instance_run_file.setup() tool_instance_run_file.components = [mock_component.from_name.return_value] tool_instance_run_file.start(n_events=10) + tool_instance_run_file.finish() assert tool_instance_run_file._n_traited_events == 10 + @patch("nectarchain.makers.core.Component") @patch( @@ -370,7 +376,7 @@ def test_finish_components(self, mock_finish_components, tool_instance_run_file) tool_instance_run_file.setup() tool_instance_run_file.components = [MockComponent()] - output = tool_instance_run_file._finish_components() + _ = tool_instance_run_file._finish_components() assert mock_finish_components.called_with([MockComponent().finish()], 0) From e944206b95065946ad6779b86900133cae9f9cb9 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 19:31:34 +0100 Subject: [PATCH 21/36] formatting --- .../makers/component/FlatFieldSPEComponent.py | 17 ++-- .../makers/component/PedestalComponent.py | 52 ++++------ .../makers/component/chargesComponent.py | 57 ++++------- src/nectarchain/makers/component/core.py | 75 ++++++--------- .../makers/component/gainComponent.py | 4 +- .../component/photostatistic_algorithm.py | 54 ++++------- .../makers/component/spe/spe_algorithm.py | 95 +++++++------------ .../makers/component/waveformsComponent.py | 63 ++++++------ 8 files changed, 166 insertions(+), 251 deletions(-) diff --git a/src/nectarchain/makers/component/FlatFieldSPEComponent.py b/src/nectarchain/makers/component/FlatFieldSPEComponent.py index 4c72452f..ab7a5b2f 100644 --- a/src/nectarchain/makers/component/FlatFieldSPEComponent.py +++ b/src/nectarchain/makers/component/FlatFieldSPEComponent.py @@ -1,9 +1,5 @@ import logging -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import copy import numpy as np @@ -22,6 +18,10 @@ SPEnominalStdalgorithm, ) +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + __all__ = [ "FlatFieldSingleNominalSPEStdNectarCAMComponent", "FlatFieldSingleNominalSPENectarCAMComponent", @@ -47,12 +47,14 @@ class FlatFieldSingleNominalSPENectarCAMComponent(GainNectarCAMComponent): # Windows_lenght = Integer(40, # read_only = True, - # help = "The windows leght used for the savgol filter algorithm", + # help = "The windows leght used for the savgol + # filter algorithm", # ).tag(config = True) # # Order = Integer(2, # read_only = True, - # help = "The order of the polynome used in the savgol filter algorithm", + # help = "The order of the polynome used in the savgol + # filter algorithm", # ).tag(config = True) asked_pixels_id = List( @@ -79,7 +81,8 @@ class FlatFieldSingleNominalSPENectarCAMComponent(GainNectarCAMComponent): # ).tag(config = True) # # extractor_kwargs = Dict(default_value = {}, - # help = "The kwargs to be pass to the charge extractor method", + # help = "The kwargs to be pass to the charge extractor + # method", # ).tag(config = True) # constructor diff --git a/src/nectarchain/makers/component/PedestalComponent.py b/src/nectarchain/makers/component/PedestalComponent.py index 7489342d..ad255c78 100644 --- a/src/nectarchain/makers/component/PedestalComponent.py +++ b/src/nectarchain/makers/component/PedestalComponent.py @@ -24,12 +24,10 @@ class PedestalEstimationComponent(NectarCAMComponent): - """ - Component that computes calibration pedestal coefficients from raw data. - Waveforms can be filtered based on time, standard deviation of the waveforms - or charge distribution within the sample. - Use the ``events_per_slice`` parameter of ``NectarCAMComponent`` to reduce - memory load. + """Component that computes calibration pedestal coefficients from raw data. + Waveforms can be filtered based on time, standard deviation of the waveforms or + charge distribution within the sample. Use the ``events_per_slice`` parameter of + ``NectarCAMComponent`` to reduce memory load. Parameters ---------- @@ -62,7 +60,6 @@ class PedestalEstimationComponent(NectarCAMComponent): pixel_mask_std_pixel_max : float Maximum value of pedestal standard deviation in a pixel above which the pixel is flagged as bad - """ ucts_tmin = Integer( @@ -152,12 +149,10 @@ class PedestalEstimationComponent(NectarCAMComponent): SubComponents.read_only = True def __init__(self, subarray, config=None, parent=None, *args, **kwargs): - """ - Component that computes calibration pedestal coefficients from raw data. - Waveforms can be filtered based on time, standard deviation of the waveforms - or charge distribution within the sample. - Use the ``events_per_slice`` parameter of ``NectarCAMComponent`` to - reduce memory load. + """Component that computes calibration pedestal coefficients from raw data. + Waveforms can be filtered based on time, standard deviation of the waveforms or + charge distribution within the sample. Use the ``events_per_slice`` parameter of + ``NectarCAMComponent`` to reduce memory load. Parameters ---------- @@ -208,8 +203,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): @staticmethod def calculate_stats(waveformsContainers, wfs_mask, statistics): - """ - Calculate statistics for the pedestals from a waveforms container. + """Calculate statistics for the pedestals from a waveforms container. Parameters ---------- @@ -250,8 +244,7 @@ def calculate_stats(waveformsContainers, wfs_mask, statistics): return ped_stats def flag_bad_pixels(self, ped_stats, nevents): - """ - Flag bad pixels based on pedestal properties + """Flag bad pixels based on pedestal properties. Parameters ---------- @@ -326,9 +319,7 @@ def flag_bad_pixels(self, ped_stats, nevents): return pixel_mask def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): - """ - Fill the waveform container looping over the events of type SKY_PEDESTAL. - """ + """Fill the waveform container looping over the events of type SKY_PEDESTAL.""" if event.trigger.event_type == EventType.SKY_PEDESTAL: self.waveformsComponent(event=event, *args, **kwargs) @@ -336,8 +327,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): pass def timestamp_mask(self, tmin, tmax): - """ - Generates a mask to filter waveforms outside the required time interval + """Generates a mask to filter waveforms outside the required time interval. Parameters ---------- @@ -383,10 +373,8 @@ def timestamp_mask(self, tmin, tmax): return new_mask def waveformsStdFilter_mask(self, threshold): - """ - Generates a mask to filter waveforms that have a standard deviation above - a threshold. - This option is effective for dark room verification data. + """Generates a mask to filter waveforms that have a standard deviation above a + threshold. This option is effective for dark room verification data. Parameters ---------- @@ -428,10 +416,8 @@ def waveformsStdFilter_mask(self, threshold): return new_mask def chargeDistributionFilter_mask(self, sigma_low, sigma_high): - """ - Generates a mask to filter waveforms that have a charge in the tails of the - distribution. - This option is useful for data with NSB. + """Generates a mask to filter waveforms that have a charge in the tails of the + distribution. This option is useful for data with NSB. Parameters ---------- @@ -501,10 +487,8 @@ def chargeDistributionFilter_mask(self, sigma_low, sigma_high): return new_mask def finish(self, *args, **kwargs): - """ - Finish the component by filtering the waveforms and calculating the pedestal - quantities. - """ + """Finish the component by filtering the waveforms and calculating the pedestal + quantities.""" # Use only pedestal type events waveformsContainers = self.waveformsComponent.finish() diff --git a/src/nectarchain/makers/component/chargesComponent.py b/src/nectarchain/makers/component/chargesComponent.py index 0b997499..d93d7309 100644 --- a/src/nectarchain/makers/component/chargesComponent.py +++ b/src/nectarchain/makers/component/chargesComponent.py @@ -61,8 +61,7 @@ cache=True, ) def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): - """ - Compute histogram of charge with numba + """Compute histogram of charge with numba. Parameters ---------- @@ -71,7 +70,6 @@ def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): mask_broken_pix (np.ndarray(pixels)): mask on broxen pixels _mask (np.ndarray(pixels,nbins)): mask hist_ma_data (np.ndarray(pixels,nbins)): histogram - """ # print(f"charge.shape = {charge.shape[0]}") # print(f"_mask.shape = {_mask.shape[0]}") @@ -138,8 +136,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): self.__peak_lg = {} def _init_trigger_type(self, trigger_type: EventType, **kwargs): - """ - Initializes the ChargesMaker based on the trigger type. + """Initializes the ChargesMaker based on the trigger type. Parameters ---------- @@ -149,7 +146,6 @@ def _init_trigger_type(self, trigger_type: EventType, **kwargs): Returns ------- None - """ super()._init_trigger_type(trigger_type, **kwargs) name = __class__._get_name_trigger(trigger_type) @@ -212,8 +208,7 @@ def _get_extractor_kwargs_from_method_and_kwargs(method: str, kwargs: dict): @staticmethod def _get_imageExtractor(method: str, subarray: SubarrayDescription, **kwargs): - """ - Create an instance of a charge extraction method based on the provided method + """Create an instance of a charge extraction method based on the provided method name and subarray description. Parameters @@ -250,8 +245,7 @@ def _get_imageExtractor(method: str, subarray: SubarrayDescription, **kwargs): return imageExtractor def finish(self, *args, **kwargs): - """ - Create an output container for the specified trigger type and method. + """Create an output container for the specified trigger type and method. Parameters ---------- @@ -263,7 +257,6 @@ def finish(self, *args, **kwargs): Returns ------- list: A list of ChargesContainer objects. - """ output = ChargesContainers() for i, trigger in enumerate(self.trigger_list): @@ -296,8 +289,7 @@ def finish(self, *args, **kwargs): @staticmethod def sort(chargesContainer: ChargesContainer, method: str = "event_id"): - """ - Sorts the charges in a ChargesContainer object based on the specified method. + """Sorts the charges in a ChargesContainer object based on the specified method. Parameters ---------- @@ -346,9 +338,8 @@ def sort(chargesContainer: ChargesContainer, method: str = "event_id"): @staticmethod def select_charges_hg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): - """ - Selects the charges from the ChargesContainer object for the given pixel_id and - returns the result transposed. + """Selects the charges from the ChargesContainer object for the given pixel_id + and returns the result transposed. Parameters ---------- @@ -371,9 +362,8 @@ def select_charges_hg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): @staticmethod def select_charges_lg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): - """ - Selects the charges from the ChargesContainer object for the given pixel_id and - returns the result transposed. + """Selects the charges from the ChargesContainer object for the given pixel_id + and returns the result transposed. Parameters ---------- @@ -395,8 +385,7 @@ def select_charges_lg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): return res def charges_hg(self, trigger: EventType): - """ - Returns the charges for a specific trigger type as a NumPy array of unsigned + """Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Parameters @@ -415,8 +404,7 @@ def charges_hg(self, trigger: EventType): ) def charges_lg(self, trigger: EventType): - """ - Returns the charges for a specific trigger type as a NumPy array of unsigned + """Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Parameters @@ -435,8 +423,7 @@ def charges_lg(self, trigger: EventType): ) def peak_hg(self, trigger: EventType): - """ - Returns the peak charges for a specific trigger type as a NumPy array of + """Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Parameters @@ -455,8 +442,7 @@ def peak_hg(self, trigger: EventType): ) def peak_lg(self, trigger: EventType): - """ - Returns the peak charges for a specific trigger type as a NumPy array of + """Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Parameters @@ -492,8 +478,7 @@ def create_from_waveforms( method: str = "FullWaveformSum", **kwargs, ) -> ChargesContainer: - """ - Create a ChargesContainer object from waveforms using the specified charge + """Create a ChargesContainer object from waveforms using the specified charge extraction method. Parameters @@ -546,8 +531,7 @@ def compute_charges( tel_id: int = None, **kwargs, ): - """ - Compute charge from waveforms. + """Compute charge from waveforms. Parameters ---------- @@ -626,8 +610,7 @@ def compute_charges( def histo_hg( chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True ) -> ma.masked_array: - """ - Computes histogram of high gain charges from a ChargesContainer object. + """Computes histogram of high gain charges from a ChargesContainer object. Parameters ---------- @@ -657,8 +640,7 @@ def histo_hg( def histo_lg( chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True ) -> ma.masked_array: - """ - Computes histogram of low gain charges from a ChargesContainer object. + """Computes histogram of low gain charges from a ChargesContainer object. Parameters ---------- @@ -691,9 +673,8 @@ def _histo( n_bins: int = 1000, autoscale: bool = True, ) -> ma.masked_array: - """ - Computes histogram of charges for a given field from a ChargesContainer object. - Numba is used to compute histograms in a vectorized way. + """Computes histogram of charges for a given field from a ChargesContainer + object. Numba is used to compute histograms in a vectorized way. Parameters ---------- diff --git a/src/nectarchain/makers/component/core.py b/src/nectarchain/makers/component/core.py index 1966a2a4..d5f88f16 100644 --- a/src/nectarchain/makers/component/core.py +++ b/src/nectarchain/makers/component/core.py @@ -30,7 +30,7 @@ def get_valid_component(): class NectarCAMComponent(TelescopeComponent): - """The base class for NectarCAM components""" + """The base class for NectarCAM components.""" SubComponents = ComponentNameList( Component, @@ -118,8 +118,8 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): self.__broken_pixels_lg = {} def _init_trigger_type(self, trigger: EventType, **kwargs): - """ - Initializes empty lists for different trigger types in the ArrayDataMaker class. + """Initializes empty lists for different trigger types in the ArrayDataMaker + class. Args: trigger (EventType): The trigger type for which the lists are being @@ -141,8 +141,8 @@ def _init_trigger_type(self, trigger: EventType, **kwargs): @staticmethod def _compute_broken_pixels(wfs_hg, wfs_lg, **kwargs): - """ - Computes broken pixels for high and low gain waveforms. + """Computes broken pixels for high and low gain waveforms. + Args: wfs_hg (ndarray): High gain waveforms. wfs_lg (ndarray): Low gain waveforms. @@ -160,8 +160,8 @@ def _compute_broken_pixels(wfs_hg, wfs_lg, **kwargs): def _compute_broken_pixels_event( event: NectarCAMDataContainer, pixels_id: np.ndarray, **kwargs ): - """ - Computes broken pixels for a specific event and pixel IDs. + """Computes broken pixels for a specific event and pixel IDs. + Args: event (NectarCAMDataContainer): An event. pixels_id (list or np.ndarray): IDs of pixels. @@ -176,8 +176,8 @@ def _compute_broken_pixels_event( @staticmethod def _get_name_trigger(trigger: EventType): - """ - Gets the name of a trigger event. + """Gets the name of a trigger event. + Args: trigger (EventType): A trigger event. Returns: @@ -190,8 +190,7 @@ def _get_name_trigger(trigger: EventType): return name def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): - """ - Method to extract data from the event. + """Method to extract data from the event. Parameters ---------- @@ -251,9 +250,8 @@ def finish(self): def select_container_array_field( container: ArrayDataContainer, pixel_id: np.ndarray, field: str ) -> np.ndarray: - """ - Selects specific fields from an ArrayDataContainer object based on a given list - of pixel IDs. + """Selects specific fields from an ArrayDataContainer object based on a given + list of pixel IDs. Args: container (ArrayDataContainer): An object of type ArrayDataContainer that @@ -313,7 +311,7 @@ def merge_along_slices( def merge( container_a: ArrayDataContainer, container_b: ArrayDataContainer ) -> ArrayDataContainer: - """method to merge 2 ArrayDataContainer into one single ArrayDataContainer + """Method to merge 2 ArrayDataContainer into one single ArrayDataContainer. Returns: ArrayDataContainer: the merged object @@ -352,8 +350,7 @@ def merge( @property def nsamples(self): - """ - Returns a deep copy of the nsamples attribute. + """Returns a deep copy of the nsamples attribute. Returns: np.ndarray: A deep copy of the nsamples attribute. @@ -362,8 +359,7 @@ def nsamples(self): @property def _nsamples(self): - """ - Returns the nsamples attribute. + """Returns the nsamples attribute. Returns: np.ndarray: The nsamples attribute. @@ -371,8 +367,7 @@ def _nsamples(self): return self.__nsamples def nevents(self, trigger: EventType): - """ - Returns the number of events for the specified trigger type. + """Returns the number of events for the specified trigger type. Args: trigger (EventType): The trigger type for which the number of events is @@ -387,8 +382,7 @@ def nevents(self, trigger: EventType): @property def _broken_pixels_hg(self): - """ - Returns the broken_pixels_hg attribute. + """Returns the broken_pixels_hg attribute. Returns: np.ndarray: The broken_pixels_hg attribute. @@ -396,8 +390,8 @@ def _broken_pixels_hg(self): return self.__broken_pixels_hg def broken_pixels_hg(self, trigger: EventType): - """ - Returns an array of broken pixels for high gain for the specified trigger type. + """Returns an array of broken pixels for high gain for the specified trigger + type. Args: trigger (EventType): The trigger type for which the broken pixels for high @@ -414,8 +408,7 @@ def broken_pixels_hg(self, trigger: EventType): @property def _broken_pixels_lg(self): - """ - Returns the broken_pixels_lg attribute. + """Returns the broken_pixels_lg attribute. Returns: np.ndarray: The broken_pixels_lg attribute. @@ -423,8 +416,8 @@ def _broken_pixels_lg(self): return self.__broken_pixels_lg def broken_pixels_lg(self, trigger: EventType): - """ - Returns an array of broken pixels for low gain for the specified trigger type. + """Returns an array of broken pixels for low gain for the specified trigger + type. Args: trigger (EventType): The trigger type for which the broken pixels for low @@ -440,8 +433,7 @@ def broken_pixels_lg(self, trigger: EventType): ) def ucts_timestamp(self, trigger: EventType): - """ - Returns an array of UCTS timestamps for the specified trigger type. + """Returns an array of UCTS timestamps for the specified trigger type. Args: trigger (EventType): The trigger type for which the UCTS timestamps are @@ -456,8 +448,7 @@ def ucts_timestamp(self, trigger: EventType): ) def ucts_busy_counter(self, trigger: EventType): - """ - Returns an array of UCTS busy counters for the specified trigger type. + """Returns an array of UCTS busy counters for the specified trigger type. Args: trigger (EventType): The trigger type for which the UCTS busy counters are @@ -472,8 +463,7 @@ def ucts_busy_counter(self, trigger: EventType): ) def ucts_event_counter(self, trigger: EventType): - """ - Returns an array of UCTS event counters for the specified trigger type. + """Returns an array of UCTS event counters for the specified trigger type. Args: trigger (EventType): The trigger type for which the UCTS event counters are @@ -488,8 +478,7 @@ def ucts_event_counter(self, trigger: EventType): ) def event_type(self, trigger: EventType): - """ - Returns an array of event types for the specified trigger type. + """Returns an array of event types for the specified trigger type. Args: trigger (EventType): The trigger type for which the event types are @@ -504,8 +493,7 @@ def event_type(self, trigger: EventType): ) def event_id(self, trigger: EventType): - """ - Returns an array of event IDs for the specified trigger type. + """Returns an array of event IDs for the specified trigger type. Args: trigger (EventType): The trigger type for which the event IDs are requested. @@ -519,8 +507,7 @@ def event_id(self, trigger: EventType): ) def multiplicity(self, trigger: EventType): - """ - Returns an array of multiplicities for the specified trigger type. + """Returns an array of multiplicities for the specified trigger type. Args: trigger (EventType): The trigger type for which the multiplicities are @@ -538,8 +525,7 @@ def multiplicity(self, trigger: EventType): ) def trig_pattern(self, trigger: EventType): - """ - Returns an array of trigger patterns for the specified trigger type. + """Returns an array of trigger patterns for the specified trigger type. Args: trigger (EventType): The trigger type for which the trigger patterns are @@ -555,8 +541,7 @@ def trig_pattern(self, trigger: EventType): return tmp.any(axis=2) def trig_pattern_all(self, trigger: EventType): - """ - Returns an array of trigger patterns for all events for the specified trigger + """Returns an array of trigger patterns for all events for the specified trigger type. Args: diff --git a/src/nectarchain/makers/component/gainComponent.py b/src/nectarchain/makers/component/gainComponent.py index 60a7c1fd..29bf328f 100644 --- a/src/nectarchain/makers/component/gainComponent.py +++ b/src/nectarchain/makers/component/gainComponent.py @@ -1,12 +1,12 @@ import logging +from abc import abstractmethod +from .core import NectarCAMComponent logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from abc import abstractmethod -from .core import NectarCAMComponent __all__ = ["GainNectarCAMComponent"] diff --git a/src/nectarchain/makers/component/photostatistic_algorithm.py b/src/nectarchain/makers/component/photostatistic_algorithm.py index c9e45743..13b552ef 100644 --- a/src/nectarchain/makers/component/photostatistic_algorithm.py +++ b/src/nectarchain/makers/component/photostatistic_algorithm.py @@ -115,10 +115,8 @@ def __get_charges_FF_Ped_reshaped( return out def __check_shape(self) -> None: - """ - Checks the shape of certain attributes and raises an exception if the shape is - not as expected. - """ + """Checks the shape of certain attributes and raises an exception if the shape + is not as expected.""" try: self.__FFcharge_hg[0] * self.__FFcharge_lg[0] * self.__Pedcharge_hg[ 0 @@ -157,8 +155,7 @@ def run(self, pixels_id: np.ndarray = None, **kwargs) -> None: def plot_correlation( photoStat_gain: np.ndarray, SPE_gain: np.ndarray ) -> plt.Figure: - """ - Plot the correlation between the photo statistic gain and the single + """Plot the correlation between the photo statistic gain and the single photoelectron (SPE) gain. Args: @@ -212,8 +209,7 @@ def plot_correlation( @property def SPE_resolution(self) -> float: - """ - Returns a deep copy of the SPE resolution. + """Returns a deep copy of the SPE resolution. Returns: float: The SPE resolution. @@ -222,9 +218,8 @@ def SPE_resolution(self) -> float: @property def sigmaPedHG(self) -> float: - """ - Calculates and returns the standard deviation of Pedcharge_hg multiplied by the - square root of coefCharge_FF_Ped. + """Calculates and returns the standard deviation of Pedcharge_hg multiplied by + the square root of coefCharge_FF_Ped. Returns: float: The standard deviation of Pedcharge_hg. @@ -233,8 +228,7 @@ def sigmaPedHG(self) -> float: @property def sigmaChargeHG(self) -> float: - """ - Calculates and returns the standard deviation of FFcharge_hg minus meanPedHG. + """Calculates and returns the standard deviation of FFcharge_hg minus meanPedHG. Returns: float: The standard deviation of FFcharge_hg minus meanPedHG. @@ -243,8 +237,8 @@ def sigmaChargeHG(self) -> float: @property def meanPedHG(self) -> float: - """ - Calculates and returns the mean of Pedcharge_hg multiplied by coefCharge_FF_Ped. + """Calculates and returns the mean of Pedcharge_hg multiplied by + coefCharge_FF_Ped. Returns: float: The mean of Pedcharge_hg. @@ -253,8 +247,7 @@ def meanPedHG(self) -> float: @property def meanChargeHG(self) -> float: - """ - Calculates and returns the mean of FFcharge_hg minus meanPedHG. + """Calculates and returns the mean of FFcharge_hg minus meanPedHG. Returns: float: The mean of FFcharge_hg minus meanPedHG. @@ -263,8 +256,7 @@ def meanChargeHG(self) -> float: @property def BHG(self) -> float: - """ - Calculates and returns the BHG value. + """Calculates and returns the BHG value. Returns: float: The BHG value. @@ -284,8 +276,7 @@ def BHG(self) -> float: @property def gainHG(self) -> float: - """ - Calculates and returns the gain for high gain charge data. + """Calculates and returns the gain for high gain charge data. Returns: float: The gain for high gain charge data. @@ -298,9 +289,8 @@ def gainHG(self) -> float: @property def sigmaPedLG(self) -> float: - """ - Calculates and returns the standard deviation of Pedcharge_lg multiplied by the - square root of coefCharge_FF_Ped. + """Calculates and returns the standard deviation of Pedcharge_lg multiplied by + the square root of coefCharge_FF_Ped. Returns: float: The standard deviation of Pedcharge_lg. @@ -309,8 +299,7 @@ def sigmaPedLG(self) -> float: @property def sigmaChargeLG(self) -> float: - """ - Calculates and returns the standard deviation of FFcharge_lg minus meanPedLG. + """Calculates and returns the standard deviation of FFcharge_lg minus meanPedLG. Returns: float: The standard deviation of FFcharge_lg minus meanPedLG. @@ -319,8 +308,8 @@ def sigmaChargeLG(self) -> float: @property def meanPedLG(self) -> float: - """ - Calculates and returns the mean of Pedcharge_lg multiplied by coefCharge_FF_Ped. + """Calculates and returns the mean of Pedcharge_lg multiplied by + coefCharge_FF_Ped. Returns: float: The mean of Pedcharge_lg. @@ -329,8 +318,7 @@ def meanPedLG(self) -> float: @property def meanChargeLG(self) -> float: - """ - Calculates and returns the mean of FFcharge_lg minus meanPedLG. + """Calculates and returns the mean of FFcharge_lg minus meanPedLG. Returns: float: The mean of FFcharge_lg minus meanPedLG. @@ -339,8 +327,7 @@ def meanChargeLG(self) -> float: @property def BLG(self) -> float: - """ - Calculates and returns the BLG value. + """Calculates and returns the BLG value. Returns: float: The BLG value. @@ -360,8 +347,7 @@ def BLG(self) -> float: @property def gainLG(self) -> float: - """ - Calculates and returns the gain for low gain charge data. + """Calculates and returns the gain for low gain charge data. Returns: float: The gain for low gain charge data. diff --git a/src/nectarchain/makers/component/spe/spe_algorithm.py b/src/nectarchain/makers/component/spe/spe_algorithm.py index ee9ff12e..46ed5d90 100644 --- a/src/nectarchain/makers/component/spe/spe_algorithm.py +++ b/src/nectarchain/makers/component/spe/spe_algorithm.py @@ -71,9 +71,7 @@ def __exit__(self, type, value, traceback): def init_processes( _class, minuitParameters_array: np.ndarray, charge: np.ndarray, counts: np.ndarray ): - """ - Initialize each process in the process pool with global variable fit_array. - """ + """Initialize each process in the process pool with global variable fit_array.""" global _minuitParameters_array global _charge global _counts @@ -168,8 +166,7 @@ def npixels(self): # methods def read_param_from_yaml(self, parameters_file, only_update=False) -> None: - """ - Reads parameters from a YAML file and updates the internal parameters of the + """Reads parameters from a YAML file and updates the internal parameters of the FlatFieldSPEMaker class. Parameters @@ -210,9 +207,8 @@ def read_param_from_yaml(self, parameters_file, only_update=False) -> None: def _update_parameters( parameters: Parameters, charge: np.ndarray, counts: np.ndarray, **kwargs ) -> Parameters: - """ - Update the parameters of the FlatFieldSPEMaker class based on the input charge - and counts data. + """Update the parameters of the FlatFieldSPEMaker class based on the input + charge and counts data. Parameters ---------- @@ -269,8 +265,7 @@ def _update_parameters( def _get_mean_gaussian_fit( charge: np.ndarray, counts: np.ndarray, pixel_id=None, **kwargs ) -> Tuple[np.ndarray, np.ndarray]: - """ - Perform a Gaussian fit on the data to determine the pedestal and mean values. + """Perform a Gaussian fit on the data to determine the pedestal and mean values. Parameters ---------- @@ -465,8 +460,7 @@ def __init__( parent=None, **kwargs, ) -> None: - """ - Initializes the FlatFieldSingleHHVSPEMaker object. + """Initializes the FlatFieldSingleHHVSPEMaker object. Parameters ---------- @@ -495,9 +489,8 @@ def __init__( def create_from_chargesContainer( cls, signal: ChargesContainer, config=None, parent=None, **kwargs ): - """ - Creates an instance of FlatFieldSingleHHVSPEMaker using charge and counts data - from a ChargesContainer object. + """Creates an instance of FlatFieldSingleHHVSPEMaker using charge and counts + data from a ChargesContainer object. Parameters ---------- @@ -524,39 +517,30 @@ def create_from_chargesContainer( # getters and setters @property def charge(self): - """ - Returns a deep copy of the ``__charge`` attribute. - """ + """Returns a deep copy of the ``__charge`` attribute.""" return copy.deepcopy(self.__charge) @property def _charge(self): - """ - Returns the ``__charge`` attribute. - """ + """Returns the ``__charge`` attribute.""" return self.__charge @property def counts(self): - """ - Returns a deep copy of the ``__counts`` attribute. - """ + """Returns a deep copy of the ``__counts`` attribute.""" return copy.deepcopy(self.__counts) @property def _counts(self): - """ - Returns the ``__counts`` attribute. - """ + """Returns the ``__counts`` attribute.""" return self.__counts # methods def _fill_results_table_from_dict( self, dico: dict, pixels_id: np.ndarray, return_fit_array: bool = True ) -> None: - """ - Populates the results table with fit values and errors for each pixel based on - the dictionary provided as input. + """Populates the results table with fit values and errors for each pixel based + on the dictionary provided as input. Parameters ---------- @@ -627,9 +611,8 @@ def _NG_Likelihood_Chi2( counts: np.ndarray, **kwargs, ): - """ - Calculates the chi-square value using the MPE2 function. - The different parameters are explained in `Caroff et al. (2019) `_. + """Calculates the chi-square value using the MPE2 function. The different + parameters are explained in `Caroff et al. (2019) `_. .. _CAROFF: https://ui.adsabs.harvard.edu/abs/2019SPIE11119E..1WC @@ -681,8 +664,7 @@ def _NG_Likelihood_Chi2( def _make_minuitParameters_array_from_parameters( self, pixels_id: np.ndarray = None, **kwargs ) -> np.ndarray: - """ - Create an array of Minuit fit instances based on the parameters and data for + """Create an array of Minuit fit instances based on the parameters and data for each pixel. Parameters @@ -726,8 +708,7 @@ def _make_minuitParameters_array_from_parameters( @staticmethod def run_fit(i: int, tol: float) -> dict: - """ - Perform a fit on a specific pixel using the Minuit package. + """Perform a fit on a specific pixel using the Minuit package. Parameters ---------- @@ -966,9 +947,8 @@ def plot_single_matplotlib( likelihood: float, **kwargs, ) -> tuple: - """ - Generate a plot of the data and a model fit for a specific pixel. - The different parameters are explained in `Caroff et al. (2019) `_. + """Generate a plot of the data and a model fit for a specific pixel. The + different parameters are explained in `Caroff et al. (2019) `_. .. _CAROFF: https://ui.adsabs.harvard.edu/abs/2019SPIE11119E..1WC @@ -1003,7 +983,6 @@ def plot_single_matplotlib( ------- : tuple A tuple containing the generated plot figure and the axes of the plot. - """ if kwargs.get("ax", False) and kwargs.get("fig", False): fig = kwargs.get("fig") @@ -1037,8 +1016,7 @@ def plot_single_matplotlib( return fig, ax def display(self, pixels_id: np.ndarray, package="pyqtgraph", **kwargs) -> None: - """ - Display and save the plot for each specified pixel ID. + """Display and save the plot for each specified pixel ID. Parameters ---------- @@ -1114,9 +1092,7 @@ def display(self, pixels_id: np.ndarray, package="pyqtgraph", **kwargs) -> None: class SPEHHValgorithm(SPEnominalalgorithm): - """ - Class to perform fit of the SPE HHV signal with ``n`` and ``pp`` free. - """ + """Class to perform fit of the SPE HHV signal with ``n`` and ``pp`` free.""" parameters_file = Unicode( "parameters_SPEHHV.yaml", @@ -1131,7 +1107,7 @@ class SPEHHValgorithm(SPEnominalalgorithm): class SPEnominalStdalgorithm(SPEnominalalgorithm): - """Class to perform fit of the SPE signal with ``n`` and ``pp`` fixed""" + """Class to perform fit of the SPE signal with ``n`` and ``pp`` fixed.""" parameters_file = Unicode( "parameters_SPEnominalStd.yaml", @@ -1148,8 +1124,7 @@ def __init__( parent=None, **kwargs, ) -> None: - """ - Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. + """Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. Parameters ---------- @@ -1173,10 +1148,8 @@ def __init__( self.__fix_parameters() def __fix_parameters(self) -> None: - """ - Fixes the values of the ``n`` and ``pp`` parameters by setting their frozen - attribute to True. - """ + """Fixes the values of the ``n`` and ``pp`` parameters by setting their frozen + attribute to True.""" self.log.info("updating parameters by fixing pp and n") pp = self._parameters["pp"] pp.frozen = True @@ -1229,8 +1202,7 @@ def __init__( parent=None, **kwargs, ) -> None: - """ - Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. + """Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. Parameters ---------- @@ -1273,8 +1245,7 @@ def __init__( ) def __fix_parameters(self) -> None: - """ - Fixes the parameters ``n``, ``pp``, ``res``, and possibly ``luminosity``. + """Fixes the parameters ``n``, ``pp``, ``res``, and possibly ``luminosity``. Parameters ---------- @@ -1296,9 +1267,8 @@ def __fix_parameters(self) -> None: def _make_minuitParameters_array_from_parameters( self, pixels_id: np.ndarray = None, **kwargs ) -> np.ndarray: - """ - Generates the fit array from the fixed parameters and the fitted data obtained - from a 1400V run. + """Generates the fit array from the fixed parameters and the fitted data + obtained from a 1400V run. Parameters ---------- @@ -1327,9 +1297,8 @@ def _update_parameters( nectarGainSPEresult: QTable, **kwargs, ): - """ - Updates the parameters with the fixed values from the fitted data obtained from - a 1400V run. + """Updates the parameters with the fixed values from the fitted data obtained + from a 1400V run. Parameters ---------- diff --git a/src/nectarchain/makers/component/waveformsComponent.py b/src/nectarchain/makers/component/waveformsComponent.py index 50460408..b4b9c62f 100644 --- a/src/nectarchain/makers/component/waveformsComponent.py +++ b/src/nectarchain/makers/component/waveformsComponent.py @@ -1,10 +1,4 @@ import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - - import copy from argparse import ArgumentError @@ -17,6 +11,12 @@ from ...data.container import WaveformsContainer, WaveformsContainers from .core import ArrayDataComponent +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + + __all__ = ["WaveformsComponent"] @@ -47,7 +47,8 @@ def create_from_events_list( """Create a container for the extracted waveforms from a list of events. Args: - events_list (list[NectarCAMDataContainer]): A list of events to extract waveforms from. + events_list (list[NectarCAMDataContainer]): A list of events to extract + waveforms from. run_number (int): The ID of the run to be loaded. npixels (int): The number of pixels in the waveforms. nsamples (int): The number of samples in the waveforms. @@ -55,7 +56,8 @@ def create_from_events_list( pixels_id (int): The ID of the pixels to extract waveforms from. Returns: - WaveformsContainer: A container object that contains the extracted waveforms and other relevant information. + WaveformsContainer: A container object that contains the extracted waveforms + and other relevant information. """ if tel_id is None: tel_id = __class__.TEL_ID.default_value @@ -116,7 +118,6 @@ def _init_trigger_type(self, trigger_type: EventType, **kwargs): Args: trigger_type: The type of trigger. - """ super()._init_trigger_type(trigger_type, **kwargs) name = __class__._get_name_trigger(trigger_type) @@ -130,9 +131,9 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): """Process an event and extract waveforms. Args: - event (NectarCAMDataContainer): The event to process and extract waveforms from. + event (NectarCAMDataContainer): The event to process and extract waveforms + from. trigger (EventType): The type of trigger for the event. - """ wfs_hg_tmp = np.zeros((self.npixels, self.nsamples), dtype=np.uint16) wfs_lg_tmp = np.zeros((self.npixels, self.nsamples), dtype=np.uint16) @@ -158,7 +159,8 @@ def finish(self, *args, **kwargs): trigger_type (EventType): The selected trigger types. Returns: - list[WaveformsContainer]: A list of output containers for the selected trigger types. + list[WaveformsContainer]: A list of output containers for the selected + trigger types. """ output = WaveformsContainers() for i, trigger in enumerate(self.trigger_list): @@ -195,7 +197,8 @@ def sort(waveformsContainer: WaveformsContainer, method: str = "event_id"): """Sort the waveformsContainer based on a specified method. Args: - waveformsContainer (WaveformsContainer): The waveformsContainer to be sorted. + waveformsContainer (WaveformsContainer): The waveformsContainer + to be sorted. method (str, optional): The sorting method. Defaults to 'event_id'. Returns: @@ -236,8 +239,10 @@ def select_waveforms_hg( """Select HIGH GAIN waveforms from the container. Args: - waveformsContainer (WaveformsContainer): The container object that contains the waveforms. - pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms from the container. + waveformsContainer (WaveformsContainer): The container object that contains + the waveforms. + pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms + from the container. Returns: np.ndarray: An array of selected waveforms from the container. @@ -255,8 +260,10 @@ def select_waveforms_lg( """Select LOW GAIN waveforms from the container. Args: - waveformsContainer (WaveformsContainer): The container object that contains the waveforms. - pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms from the container. + waveformsContainer (WaveformsContainer): The container object that contains + the waveforms. + pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms + from the container. Returns: np.ndarray: An array of selected waveforms from the container. @@ -269,8 +276,7 @@ def select_waveforms_lg( @property def _geometry(self): - """ - Returns the private __geometry attribute of the WaveformsMaker class. + """Returns the private __geometry attribute of the WaveformsMaker class. :return: The value of the private __geometry attribute. """ @@ -278,8 +284,7 @@ def _geometry(self): @property def geometry(self): - """ - Returns a deep copy of the geometry attribute. + """Returns a deep copy of the geometry attribute. Returns: A deep copy of the geometry attribute. @@ -287,11 +292,11 @@ def geometry(self): return copy.deepcopy(self.__geometry) def wfs_hg(self, trigger: EventType): - """ - Returns the waveform data for the specified trigger type. + """Returns the waveform data for the specified trigger type. Args: - trigger (EventType): The type of trigger for which the waveform data is requested. + trigger (EventType): The type of trigger for which the waveform data is + requested. Returns: An array of waveform data for the specified trigger type. @@ -302,14 +307,16 @@ def wfs_hg(self, trigger: EventType): ) def wfs_lg(self, trigger: EventType): - """ - Returns the waveform data for the specified trigger type in the low gain channel. + """Returns the waveform data for the specified trigger type in the low gain + channel. Args: - trigger (EventType): The type of trigger for which the waveform data is requested. + trigger (EventType): The type of trigger for which the waveform data is + requested. Returns: - An array of waveform data for the specified trigger type in the low gain channel. + An array of waveform data for the specified trigger type in the low gain + channel. """ return np.array( self.__wfs_lg[__class__._get_name_trigger(trigger)], From 02ee8c3ddfccf28d47416e3ad35c240dae99019f Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 19:34:24 +0100 Subject: [PATCH 22/36] formatting --- src/nectarchain/makers/component/FlatFieldSPEComponent.py | 7 +++---- src/nectarchain/makers/component/gainComponent.py | 2 +- src/nectarchain/makers/component/waveformsComponent.py | 5 ++--- src/nectarchain/makers/tests/test_core.py | 6 +++--- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/nectarchain/makers/component/FlatFieldSPEComponent.py b/src/nectarchain/makers/component/FlatFieldSPEComponent.py index ab7a5b2f..5d56aac3 100644 --- a/src/nectarchain/makers/component/FlatFieldSPEComponent.py +++ b/src/nectarchain/makers/component/FlatFieldSPEComponent.py @@ -1,6 +1,5 @@ -import logging - import copy +import logging import numpy as np from ctapipe.core.traits import List, Unicode @@ -47,13 +46,13 @@ class FlatFieldSingleNominalSPENectarCAMComponent(GainNectarCAMComponent): # Windows_lenght = Integer(40, # read_only = True, - # help = "The windows leght used for the savgol + # help = "The windows leght used for the savgol # filter algorithm", # ).tag(config = True) # # Order = Integer(2, # read_only = True, - # help = "The order of the polynome used in the savgol + # help = "The order of the polynome used in the savgol # filter algorithm", # ).tag(config = True) diff --git a/src/nectarchain/makers/component/gainComponent.py b/src/nectarchain/makers/component/gainComponent.py index 29bf328f..f62537b7 100644 --- a/src/nectarchain/makers/component/gainComponent.py +++ b/src/nectarchain/makers/component/gainComponent.py @@ -2,12 +2,12 @@ from abc import abstractmethod from .core import NectarCAMComponent + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers - __all__ = ["GainNectarCAMComponent"] diff --git a/src/nectarchain/makers/component/waveformsComponent.py b/src/nectarchain/makers/component/waveformsComponent.py index b4b9c62f..0127b5ab 100644 --- a/src/nectarchain/makers/component/waveformsComponent.py +++ b/src/nectarchain/makers/component/waveformsComponent.py @@ -1,5 +1,5 @@ -import logging import copy +import logging from argparse import ArgumentError import numpy as np @@ -11,13 +11,12 @@ from ...data.container import WaveformsContainer, WaveformsContainers from .core import ArrayDataComponent + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers - - __all__ = ["WaveformsComponent"] diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core.py index 91b5b3df..d54dc135 100644 --- a/src/nectarchain/makers/tests/test_core.py +++ b/src/nectarchain/makers/tests/test_core.py @@ -300,15 +300,15 @@ def test_start(self, mock_component, tool_instance_run_file): @patch( "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" ) - def test_start_n_events(self, mock_finish_component, - mock_component, tool_instance_run_file): + def test_start_n_events( + self, mock_finish_component, mock_component, tool_instance_run_file + ): tool_instance_run_file.overwrite = True tool_instance_run_file.setup() tool_instance_run_file.components = [mock_component.from_name.return_value] tool_instance_run_file.start(n_events=10) tool_instance_run_file.finish() assert tool_instance_run_file._n_traited_events == 10 - @patch("nectarchain.makers.core.Component") @patch( From d392b4c1bb23ba50302c2178b9c748d7439c2bcf Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 22:18:25 +0100 Subject: [PATCH 23/36] flake8 formatting --- src/nectarchain/data/__init__.py | 3 +- src/nectarchain/data/container/core.py | 25 ++++--- .../data/container/tests/test_charge.py | 3 - .../data/container/tests/test_core.py | 3 +- .../data/container/tests/test_gain.py | 4 -- .../data/container/tests/test_pedestal.py | 19 +++--- .../data/container/tests/test_waveform.py | 3 - src/nectarchain/data/tests/test_management.py | 2 +- .../makers/calibration/__init__.py | 24 ++++++- src/nectarchain/makers/calibration/core.py | 68 ++----------------- .../makers/calibration/flatfieldMakers.py | 6 +- .../calibration/gain/FlatFieldSPEMakers.py | 33 +++++---- .../makers/calibration/gain/__init__.py | 20 +++++- .../makers/calibration/gain/core.py | 7 +- .../calibration/gain/photostat_makers.py | 46 +++++++++---- .../makers/calibration/pedestalMakers.py | 16 ++--- .../calibration/tests/test_pedestal_tool.py | 3 +- src/nectarchain/makers/chargesMakers.py | 25 ++++--- .../makers/component/FlatFieldSPEComponent.py | 14 ++-- .../component/photostatistic_algorithm.py | 3 +- .../component/photostatistic_component.py | 2 +- .../makers/extractor/charge_extractor.py | 12 ++-- src/nectarchain/makers/extractor/utils.py | 7 +- .../makers/tests/test_charges_makers.py | 4 +- src/nectarchain/makers/tests/test_core.py | 12 ++-- .../makers/tests/test_waveforms_makers.py | 6 +- .../ggrolleron/gain_PhotoStat_computation.py | 3 +- 27 files changed, 192 insertions(+), 181 deletions(-) diff --git a/src/nectarchain/data/__init__.py b/src/nectarchain/data/__init__.py index c89b3978..bc094f86 100644 --- a/src/nectarchain/data/__init__.py +++ b/src/nectarchain/data/__init__.py @@ -1,4 +1,5 @@ -"""Description: This file is used to import all the classes and functions from the data module.""" +"""Description: This file is used to import all the classes and functions +from the data module.""" from .container import ( ArrayDataContainer, diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index ab645c6c..8e783f10 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -234,7 +234,8 @@ def _container_from_hdf5( slice_index (int, optional): The index of the slice of data within the hdf5 file to read. Default is None. - This method first checks if the path is a string and converts it to a Path object + This method first checks if the path is a string and converts it to a Path + object if it is. It then imports the module of the container class and creates an instance of the container class. @@ -257,7 +258,7 @@ def _container_from_hdf5( """ if isinstance(path, str): path = Path(path) - module = importlib.import_module(f"{container_class.__module__}") + _ = importlib.import_module(f"{container_class.__module__}") container = eval(f"module.{container_class.__name__}")() with HDF5TableReader(path) as reader: @@ -268,11 +269,13 @@ def _container_from_hdf5( slices, will return a generator" ) for data in reader._h5file.root.__members__: - # container.containers[data] = eval(f"module.{container_class.__name__}s")() + # container.containers[data] = + # eval(f"module.{container_class.__name__}s")() for key, trigger in EventType.__members__.items(): try: _container = eval( - f"module.{container.fields['containers'].default_factory.args[0].__name__}" + f"module." + f"{container.fields['containers'].default_factory.args[0].__name__}" # noqa ) waveforms_data = eval( f"reader._h5file.root.{data}.__members__" @@ -283,7 +286,8 @@ def _container_from_hdf5( _waveforms_data = np.array(waveforms_data)[_mask] if len(_waveforms_data) == 1: tableReader = reader.read( - table_name=f"/{data}/{_waveforms_data[0]}/{trigger.name}", + table_name=f"/{data}/\ + {_waveforms_data[0]}/{trigger.name}", containers=_container, ) # container.containers[data].containers[trigger] = @@ -292,9 +296,9 @@ def _container_from_hdf5( else: log.info( - f"there is {len(_waveforms_data)} entry corresponding \ - to a {container_class} table save,\ - unable to load" + f"there is {len(_waveforms_data)} entry\ + corresponding to a {container_class}\ + table save, unable to load" ) except NoSuchNodeError as err: log.warning(err) @@ -319,10 +323,11 @@ def _container_from_hdf5( for key, trigger in EventType.__members__.items(): try: _container = eval( - f"module.{container.fields['containers'].default_factory.args[0].__name__}" + f"module.{container.fields['containers'].default_factory.args[0].__name__}" # noqa ) tableReader = reader.read( - table_name=f"/{data}/{_container.__name__}_{index_component}/{trigger.name}", + table_name=f"/{data}/{_container.__name__}_\ + {index_component}/{trigger.name}", containers=_container, ) container.containers[trigger] = next(tableReader) diff --git a/src/nectarchain/data/container/tests/test_charge.py b/src/nectarchain/data/container/tests/test_charge.py index 2c902b1c..750b2e0c 100644 --- a/src/nectarchain/data/container/tests/test_charge.py +++ b/src/nectarchain/data/container/tests/test_charge.py @@ -1,7 +1,4 @@ -import glob - import numpy as np -import pytest from ctapipe.containers import EventType from ctapipe.io import HDF5TableWriter diff --git a/src/nectarchain/data/container/tests/test_core.py b/src/nectarchain/data/container/tests/test_core.py index 61603bb9..8c12edcc 100644 --- a/src/nectarchain/data/container/tests/test_core.py +++ b/src/nectarchain/data/container/tests/test_core.py @@ -1,6 +1,5 @@ import numpy as np -import pytest -from ctapipe.containers import Container, EventType, Field +from ctapipe.containers import EventType, Field from ctapipe.io import HDF5TableWriter from nectarchain.data.container import ( diff --git a/src/nectarchain/data/container/tests/test_gain.py b/src/nectarchain/data/container/tests/test_gain.py index 31d208ec..e356e6d7 100644 --- a/src/nectarchain/data/container/tests/test_gain.py +++ b/src/nectarchain/data/container/tests/test_gain.py @@ -1,8 +1,4 @@ -import glob - import numpy as np -import pytest -from ctapipe.containers import EventType from ctapipe.io import HDF5TableWriter from nectarchain.data.container import GainContainer, SPEfitContainer diff --git a/src/nectarchain/data/container/tests/test_pedestal.py b/src/nectarchain/data/container/tests/test_pedestal.py index 48fa3b7c..07ba89b6 100644 --- a/src/nectarchain/data/container/tests/test_pedestal.py +++ b/src/nectarchain/data/container/tests/test_pedestal.py @@ -1,7 +1,4 @@ -import tempfile - import numpy as np -from ctapipe.io import HDF5TableWriter from nectarchain.data.container import NectarCAMPedestalContainer @@ -40,7 +37,7 @@ def generate_mock_pedestal_container(): pedestal_container.validate() # create dictionary that duplicates content - dict = { + _dict = { "nsamples": nsamples, "nevents": nevents, "pixels_id": pixels_id, @@ -54,7 +51,7 @@ def generate_mock_pedestal_container(): } # return both container and input content - return pedestal_container, dict + return pedestal_container, _dict class TestNectarCAMPedestalContainer: @@ -103,10 +100,14 @@ def test_create_pedestal_container_mem(self): # assert pedestal_container.nevents.tolist() == dict['nevents'].tolist() # assert pedestal_container.ucts_timestamp_min == dict['ucts_timestamp_min'] # assert pedestal_container.ucts_timestamp_max == dict['ucts_timestamp_max'] - # assert np.allclose(pedestal_container.pedestal_mean_hg, dict['pedestal_mean_hg']) - # assert np.allclose(pedestal_container.pedestal_mean_lg, dict['pedestal_mean_lg']) - # assert np.allclose(pedestal_container.pedestal_std_hg, dict['pedestal_std_hg']) - # assert np.allclose(pedestal_container.pedestal_std_lg, dict['pedestal_std_lg']) + # assert np.allclose(pedestal_container.pedestal_mean_hg, + # dict['pedestal_mean_hg']) + # assert np.allclose(pedestal_container.pedestal_mean_lg, + # dict['pedestal_mean_lg']) + # assert np.allclose(pedestal_container.pedestal_std_hg, + # dict['pedestal_std_hg']) + # assert np.allclose(pedestal_container.pedestal_std_lg, + # dict['pedestal_std_lg']) if __name__ == "__main__": diff --git a/src/nectarchain/data/container/tests/test_waveform.py b/src/nectarchain/data/container/tests/test_waveform.py index f240c51f..01d62a9d 100644 --- a/src/nectarchain/data/container/tests/test_waveform.py +++ b/src/nectarchain/data/container/tests/test_waveform.py @@ -1,7 +1,4 @@ -import glob - import numpy as np -import pytest from ctapipe.containers import EventType from ctapipe.io import HDF5TableWriter diff --git a/src/nectarchain/data/tests/test_management.py b/src/nectarchain/data/tests/test_management.py index 5871ed8e..f77393f6 100644 --- a/src/nectarchain/data/tests/test_management.py +++ b/src/nectarchain/data/tests/test_management.py @@ -1 +1 @@ -import pytest +# To DO diff --git a/src/nectarchain/makers/calibration/__init__.py b/src/nectarchain/makers/calibration/__init__.py index 0ded83f5..92bc32c7 100644 --- a/src/nectarchain/makers/calibration/__init__.py +++ b/src/nectarchain/makers/calibration/__init__.py @@ -1,3 +1,21 @@ -from .flatfieldMakers import * -from .gain import * -from .pedestalMakers import * +from .flatfieldMakers import FlatfieldNectarCAMCalibrationTool +from .gain import ( + FlatFieldSPECombinedStdNectarCAMCalibrationTool, + FlatFieldSPEHHVNectarCAMCalibrationTool, + FlatFieldSPEHHVStdNectarCAMCalibrationTool, + FlatFieldSPENominalNectarCAMCalibrationTool, + FlatFieldSPENominalStdNectarCAMCalibrationTool, + PhotoStatisticNectarCAMCalibrationTool, +) +from .pedestalMakers import PedestalNectarCAMCalibrationTool + +__all__ = [ + "FlatfieldNectarCAMCalibrationTool", + "FlatFieldSPECombinedStdNectarCAMCalibrationTool", + "FlatFieldSPEHHVNectarCAMCalibrationTool", + "FlatFieldSPEHHVStdNectarCAMCalibrationTool", + "FlatFieldSPENominalNectarCAMCalibrationTool", + "FlatFieldSPENominalStdNectarCAMCalibrationTool", + "PedestalNectarCAMCalibrationTool", + "PhotoStatisticNectarCAMCalibrationTool", +] diff --git a/src/nectarchain/makers/calibration/core.py b/src/nectarchain/makers/calibration/core.py index c6c46e5f..05c8d2ca 100644 --- a/src/nectarchain/makers/calibration/core.py +++ b/src/nectarchain/makers/calibration/core.py @@ -1,14 +1,15 @@ import logging +from ctapipe.core.traits import List + +from ..core import EventsLoopNectarCAMCalibrationTool + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from ctapipe.core.traits import List -from ..core import EventsLoopNectarCAMCalibrationTool - -__all__ = [""] +__all__ = None class NectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): @@ -21,62 +22,3 @@ class NectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): help="the list of pixel id to apply the components", allow_none=True, ).tag(config=True) - - -''' - def setup(self) -> None: - super().setup() - self.__results = QTable() - self.__results.add_column( - Column( - self.pixels_id, - self.pixels_id.name, - unit=u.dimensionless_unscaled, - ) - ) - self.__results.meta[__class__.NP_PIXELS] = self.npixels - self.__results.meta[ - "comments" - ] = f'Produced with NectarChain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' - - def finish(self, path, **kwargs): - """ - Saves the results to a file in the specified path. - - Args: - path (str): The path to save the results. - **kwargs: Additional keyword arguments. - - Keyword Args: - overwrite (bool): Whether to overwrite an existing file. Defaults to False. - """ - path = Path(path) - path.mkdir(parents=True, exist_ok=True) - log.info(f"data saved in {path}") - self._results.write( - f"{path}/results_{self._reduced_name}.ecsv", - format="ascii.ecsv", - overwrite=kwargs.get("overwrite", False), - ) - - - @property - def _results(self): - """ - Get the result table. - - Returns: - QTable: The result table. - """ - return self.__results - - @property - def results(self): - """ - Get a copy of the result table. - - Returns: - QTable: A copy of the result table. - """ - return copy(self.__results) -''' diff --git a/src/nectarchain/makers/calibration/flatfieldMakers.py b/src/nectarchain/makers/calibration/flatfieldMakers.py index 01da9958..046ce2de 100644 --- a/src/nectarchain/makers/calibration/flatfieldMakers.py +++ b/src/nectarchain/makers/calibration/flatfieldMakers.py @@ -1,10 +1,11 @@ import logging +from .core import NectarCAMCalibrationTool + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from .core import NectarCAMCalibrationTool __all__ = ["FlatfieldNectarCAMCalibrationTool"] @@ -12,5 +13,6 @@ class FlatfieldNectarCAMCalibrationTool(NectarCAMCalibrationTool): def start(self): raise NotImplementedError( - "The computation of the flatfield calibration is not yet implemented, feel free to contribute !:)" + "The computation of the flatfield calibration is not yet implemented, \ + feel free to contribute !:)" ) diff --git a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py index 516ec85c..96654928 100644 --- a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py @@ -1,9 +1,4 @@ import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import os import pathlib @@ -18,6 +13,11 @@ from ...extractor.utils import CtapipeExtractor from .core import GainNectarCAMCalibrationTool +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = [ "FlatFieldSPENominalNectarCAMCalibrationTool", "FlatFieldSPENominalStdNectarCAMCalibrationTool", @@ -57,7 +57,8 @@ def __init__(self, *args, **kwargs): ) if len(files) == 1: log.warning( - "You asked events_per_slice but you don't want to reload events and a charges file is on disk, then events_per_slice is set to None" + "You asked events_per_slice but you don't want to reload events and\ + a charges file is on disk, then events_per_slice is set to None" ) self.events_per_slice = None @@ -70,9 +71,11 @@ def _init_output_path(self): else: ext = f"_sliced{self.events_per_slice}.h5" if self.max_events is None: - filename = f"{self.name}_run{self.run_number}_{self.method}_{str_extractor_kwargs}{ext}" + filename = f"{self.name}_run{self.run_number}_\ + {self.method}_{str_extractor_kwargs}{ext}" else: - filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}_{self.method}_{str_extractor_kwargs}{ext}" + filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}\ + _{self.method}_{str_extractor_kwargs}{ext}" self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/SPEfit/{filename}" @@ -98,7 +101,10 @@ def start( if self.reload_events or len(files) != 1: if len(files) != 1: self.log.info( - f"{len(files)} computed charges files found with max_events > {self.max_events} for run {self.run_number} with extraction method {self.method} and {str_extractor_kwargs},\n reload charges from event loop" + f"{len(files)} computed charges files found with max_events >\ + {self.max_events} for run {self.run_number} with extraction\ + method {self.method} and {str_extractor_kwargs},\n reload\ + charges from event loop" ) super().start( n_events=n_events, @@ -143,7 +149,8 @@ def start( def _write_container(self, container: Container, index_component: int = 0) -> None: # if isinstance(container,SPEfitContainer) : - # self.writer.write(table_name = f"{self.method}_{CtapipeExtractor.get_extractor_kwargs_str(self.extractor_kwargs)}", + # self.writer.write(table_name = f"{self.method}_ + # {CtapipeExtractor.get_extractor_kwargs_str(self.extractor_kwargs)}", # containers = container, # ) # else : @@ -201,9 +208,11 @@ def _init_output_path(self): self.extractor_kwargs ) if self.max_events is None: - filename = f"{self.name}_run{self.run_number}_HHV{HHVrun}_{self.method}_{str_extractor_kwargs}.h5" + filename = f"{self.name}_run{self.run_number}_HHV{HHVrun}_{self.method}_\ + {str_extractor_kwargs}.h5" else: - filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}_HHV{HHVrun}_{self.method}_{str_extractor_kwargs}.h5" + filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}_\ + HHV{HHVrun}_{self.method}_{str_extractor_kwargs}.h5" self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/SPEfit/{filename}" diff --git a/src/nectarchain/makers/calibration/gain/__init__.py b/src/nectarchain/makers/calibration/gain/__init__.py index 518e6584..44498cb4 100644 --- a/src/nectarchain/makers/calibration/gain/__init__.py +++ b/src/nectarchain/makers/calibration/gain/__init__.py @@ -1,5 +1,19 @@ -from .FlatFieldSPEMakers import * -from .photostat_makers import * +from .FlatFieldSPEMakers import ( + FlatFieldSPECombinedStdNectarCAMCalibrationTool, + FlatFieldSPEHHVNectarCAMCalibrationTool, + FlatFieldSPEHHVStdNectarCAMCalibrationTool, + FlatFieldSPENominalNectarCAMCalibrationTool, + FlatFieldSPENominalStdNectarCAMCalibrationTool, +) +from .photostat_makers import PhotoStatisticNectarCAMCalibrationTool # from .WhiteTargetSPEMakers import * -# from .PhotoStatisticMakers import * + +__all__ = [ + "FlatFieldSPENominalNectarCAMCalibrationTool", + "FlatFieldSPENominalStdNectarCAMCalibrationTool", + "FlatFieldSPEHHVNectarCAMCalibrationTool", + "FlatFieldSPEHHVStdNectarCAMCalibrationTool", + "FlatFieldSPECombinedStdNectarCAMCalibrationTool", + "PhotoStatisticNectarCAMCalibrationTool", +] diff --git a/src/nectarchain/makers/calibration/gain/core.py b/src/nectarchain/makers/calibration/gain/core.py index 095fc32c..02723e57 100644 --- a/src/nectarchain/makers/calibration/gain/core.py +++ b/src/nectarchain/makers/calibration/gain/core.py @@ -1,12 +1,13 @@ import logging +from ctapipe.core.traits import Bool + +from ..core import NectarCAMCalibrationTool + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from ctapipe.core.traits import Bool - -from ..core import NectarCAMCalibrationTool __all__ = ["GainNectarCAMCalibrationTool"] diff --git a/src/nectarchain/makers/calibration/gain/photostat_makers.py b/src/nectarchain/makers/calibration/gain/photostat_makers.py index 4a2b426d..48adba17 100644 --- a/src/nectarchain/makers/calibration/gain/photostat_makers.py +++ b/src/nectarchain/makers/calibration/gain/photostat_makers.py @@ -1,28 +1,29 @@ import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import os import pathlib import numpy as np -from ctapipe.containers import Container, EventType +from ctapipe.containers import Container from ctapipe.core.traits import ComponentNameList, Integer, Path from ....data.container import ChargesContainer, ChargesContainers -from ....data.container.core import NectarCAMContainer, merge_map_ArrayDataContainer +from ....data.container.core import merge_map_ArrayDataContainer from ....data.management import DataManagement from ...component import ArrayDataComponent, NectarCAMComponent from ...extractor.utils import CtapipeExtractor from .core import GainNectarCAMCalibrationTool +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["PhotoStatisticNectarCAMCalibrationTool"] class PhotoStatisticNectarCAMCalibrationTool(GainNectarCAMCalibrationTool): - ###TO DO : IMPLEMENT a MOTHER PHOTOSTAT CLASS WITH ONLY 1 RUN WITH FF AND PEDESTAL INTERLEAVED. + # TO DO : IMPLEMENT a MOTHER PHOTOSTAT CLASS WITH ONLY 1 RUN WITH FF AND PEDESTAL + # INTERLEAVED. name = "PhotoStatisticNectarCAM" componentsList = ComponentNameList( @@ -38,14 +39,16 @@ class PhotoStatisticNectarCAMCalibrationTool(GainNectarCAMCalibrationTool): ).tag(config=True) run_file = Path( - help="desactivated for PhotoStatistic maker with FF and pedestal runs separated", + help="desactivated for PhotoStatistic maker\ + with FF and pedestal runs separated", default_value=None, allow_none=True, read_only=True, ).tag(config=False) events_per_slice = Integer( - help="desactivated for PhotoStatistic maker with FF and pedestal runs separated", + help="desactivated for PhotoStatistic maker\ + with FF and pedestal runs separated", default_value=None, allow_none=True, read_only=True, @@ -59,9 +62,12 @@ def _init_output_path(self): self.extractor_kwargs ) if self.max_events is None: - filename = f"{self.name}_FFrun{self.run_number}_{self.method}_{str_extractor_kwargs}_Pedrun{self.Ped_run_number}_FullWaveformSum.h5" + filename = f"{self.name}_FFrun{self.run_number}_{self.method}\ + _{str_extractor_kwargs}_Pedrun{self.Ped_run_number}_FullWaveformSum.h5" else: - filename = f"{self.name}_FFrun{self.run_number}_{self.method}_{str_extractor_kwargs}_Pedrun{self.Ped_run_number}_FullWaveformSum_maxevents{self.max_events}.h5" + filename = f"{self.name}_FFrun{self.run_number}_{self.method}\ + _{str_extractor_kwargs}_Pedrun{self.Ped_run_number}_\ + FullWaveformSum_maxevents{self.max_events}.h5" self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/PhotoStat/{filename}" ) @@ -100,10 +106,19 @@ def start( if self.reload_events or len(FF_files) != 1 or len(Ped_files) != 1: if len(FF_files) != 1 or len(Ped_files) != 1: self.log.info( - f"{len(FF_files)} computed charges FF files found with max_events > {self.max_events} for run {self.run_number} with extraction method {self.method} and {str_extractor_kwargs},\n reload charges from event loop" + f"{len(FF_files)} computed charges FF files found\ + with max_events >\ + {self.max_events} for run {self.run_number}\ + with extraction method\ + {self.method} and {str_extractor_kwargs},\n reload charges\ + from event loop" ) self.log.info( - f"{len(Ped_files)} computed charges FF files found with max_events > {self.max_events} for run {self.Ped_run_number} with extraction method FullWaveformSum,\n reload charges from event loop" + f"{len(Ped_files)} computed charges FF files found\ + with max_events >\ + {self.max_events} for run {self.Ped_run_number}\ + with extraction\ + method FullWaveformSum,\n reload charges from event loop" ) super().start( @@ -184,7 +199,8 @@ def start( def _write_container(self, container: Container, index_component: int = 0) -> None: # if isinstance(container,SPEfitContainer) : - # self.writer.write(table_name = f"{self.method}_{CtapipeExtractor.get_extractor_kwargs_str(self.extractor_kwargs)}", + # self.writer.write(table_name = f"{self.method}_{CtapipeExtractor.get_extrac + # tor_kwargs_str(self.extractor_kwargs)}", # containers = container, # ) # else : diff --git a/src/nectarchain/makers/calibration/pedestalMakers.py b/src/nectarchain/makers/calibration/pedestalMakers.py index 8519ce9b..0db5386f 100644 --- a/src/nectarchain/makers/calibration/pedestalMakers.py +++ b/src/nectarchain/makers/calibration/pedestalMakers.py @@ -1,14 +1,8 @@ import logging import os - -import numpy as np - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import pathlib +import numpy as np import tables from ctapipe.core.traits import ComponentNameList from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN, N_GAINS @@ -17,6 +11,11 @@ from ..component import NectarCAMComponent from .core import NectarCAMCalibrationTool +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["PedestalNectarCAMCalibrationTool"] @@ -128,7 +127,8 @@ def _combine_results(self): pedestal_std_lg /= nevents[:, np.newaxis] pedestal_std_lg = np.sqrt(pedestal_std_lg) - # flag bad pixels in overall results based on same criteria as for individual slides + # flag bad pixels in overall results based on same criteria as for individual + # slides # reconstitute dictionary with cumulated results consistently with # PedestalComponent ped_stats = {} diff --git a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py index ec9d9822..9d7f8f6d 100644 --- a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py +++ b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py @@ -339,7 +339,8 @@ def test_pixel_mask(self): for flag_bit in flag_bits: assert np.all(output.pixel_mask & flag_bit == 0) - # For all the following tests we set the acceptable values to a range out of what + # For all the following tests we set the acceptable values to a range + # out of what # is normal. Since our test run is good we expect to flag all pixels # Condition on mean pedestal value diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/chargesMakers.py index 8662d9bc..0fbcc2eb 100644 --- a/src/nectarchain/makers/chargesMakers.py +++ b/src/nectarchain/makers/chargesMakers.py @@ -4,16 +4,14 @@ import numpy as np from ctapipe.core.traits import Bool, ComponentNameList -from ctapipe.image.extractor import ( - BaselineSubtractedNeighborPeakWindowSum, - FixedWindowSum, - FullWaveformSum, - GlobalPeakWindowSum, - LocalPeakWindowSum, - NeighborPeakWindowSum, - SlidingWindowMaxSum, - TwoPassWindowSum, -) +from ctapipe.image.extractor import BaselineSubtractedNeighborPeakWindowSum # noqa +from ctapipe.image.extractor import FixedWindowSum # noqa +from ctapipe.image.extractor import FullWaveformSum # noqa +from ctapipe.image.extractor import GlobalPeakWindowSum # noqa +from ctapipe.image.extractor import LocalPeakWindowSum # noqa +from ctapipe.image.extractor import NeighborPeakWindowSum # noqa +from ctapipe.image.extractor import SlidingWindowMaxSum # noqa +from ctapipe.image.extractor import TwoPassWindowSum # noqa from ..data.container import WaveformsContainer, WaveformsContainers from ..data.management import DataManagement @@ -71,7 +69,7 @@ def start( *args, **kwargs, ): - ##cette implémentation est complétement nulle + # cette implémentation est complétement nulle if self.from_computed_waveforms: files = DataManagement.find_waveforms( run_number=self.run_number, max_events=self.max_events @@ -90,7 +88,8 @@ def start( ) else: self.log.info( - f"{files[0]} is the computed wavforms files found with max_events >=\ + f"{files[0]} is the computed wavforms files found\ + with max_events >=\ {self.max_events} for run {self.run_number}" ) waveformsContainers = WaveformsContainers.from_hdf5(files[0]) @@ -123,7 +122,7 @@ def start( waveformsContainers ): self._init_writer(sliced=True, slice_index=slice_index) - chargesContainers = ChargesComponent._create_from_waveforms_looping_eventType( + chargesContainers = ChargesComponent._create_from_waveforms_looping_eventType( # noqa waveformsContainers=_waveformsContainers, subarray=self.event_source.subarray, method=self.method, diff --git a/src/nectarchain/makers/component/FlatFieldSPEComponent.py b/src/nectarchain/makers/component/FlatFieldSPEComponent.py index 5d56aac3..f128eeb3 100644 --- a/src/nectarchain/makers/component/FlatFieldSPEComponent.py +++ b/src/nectarchain/makers/component/FlatFieldSPEComponent.py @@ -9,13 +9,11 @@ from ...utils import ComponentUtils from .chargesComponent import ChargesComponent from .gainComponent import GainNectarCAMComponent -from .spe import ( - SPECombinedalgorithm, - SPEHHValgorithm, - SPEHHVStdalgorithm, - SPEnominalalgorithm, - SPEnominalStdalgorithm, -) +from .spe import SPECombinedalgorithm # noqa +from .spe import SPEHHValgorithm # noqa +from .spe import SPEHHVStdalgorithm # noqa +from .spe import SPEnominalalgorithm # noqa +from .spe import SPEnominalStdalgorithm # noqa logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) @@ -135,7 +133,7 @@ def finish(self, *args, **kwargs): spe_fit = eval(self.SPEfitalgorithm).create_from_chargesContainer( self._chargesContainers, parent=self, **self._SPEfitalgorithm_kwargs ) - fit_output = spe_fit.run(pixels_id=self.asked_pixels_id, *args, **kwargs) + _ = spe_fit.run(pixels_id=self.asked_pixels_id, *args, **kwargs) n_asked_pix = ( len(self._chargesContainers.pixels_id) if self.asked_pixels_id is None diff --git a/src/nectarchain/makers/component/photostatistic_algorithm.py b/src/nectarchain/makers/component/photostatistic_algorithm.py index 13b552ef..8a9c991a 100644 --- a/src/nectarchain/makers/component/photostatistic_algorithm.py +++ b/src/nectarchain/makers/component/photostatistic_algorithm.py @@ -182,7 +182,8 @@ def plot_correlation( x = np.linspace(photoStat_gain[mask].min(), photoStat_gain[mask].max(), 1000) # Define a lambda function for the linear fit line - y = lambda x: a * x + b + def y(x): + return a * x + b with quantity_support(): # Create a scatter plot of the filtered data points diff --git a/src/nectarchain/makers/component/photostatistic_component.py b/src/nectarchain/makers/component/photostatistic_component.py index 1c5ea0ed..0e3c9d79 100644 --- a/src/nectarchain/makers/component/photostatistic_component.py +++ b/src/nectarchain/makers/component/photostatistic_component.py @@ -10,7 +10,7 @@ from ...utils import ComponentUtils from .chargesComponent import ChargesComponent from .gainComponent import GainNectarCAMComponent -from .photostatistic_algorithm import PhotoStatisticAlgorithm +from .photostatistic_algorithm import PhotoStatisticAlgorithm # noqa logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) diff --git a/src/nectarchain/makers/extractor/charge_extractor.py b/src/nectarchain/makers/extractor/charge_extractor.py index 670234c5..e34bfdef 100644 --- a/src/nectarchain/makers/extractor/charge_extractor.py +++ b/src/nectarchain/makers/extractor/charge_extractor.py @@ -28,12 +28,12 @@ def __call__(self, waveforms, telid, selected_gain_channel, substract_ped=False) shape[0] * shape[1], 1 ) - y = waveforms - (ped_mean) @ ( + _ = waveforms - (ped_mean) @ ( np.ones(1, shape[2]) ) # waveforms without pedestal else: log.info("do not substract pedestal") - y = waveforms + _ = waveforms waveforms.reshape(shape[0], shape[1], shape[2]) @@ -80,15 +80,17 @@ def extract_charge(y, height_peak, fixed_window): xi[peaks[max_peak_index]] < 40 ): # Search the adaptive integration window # calculate total gradients (not used, only for plot) - yi_grad_tot = np.gradient(yi, 1) + _ = np.gradient(yi, 1) maxposition = peaks[max_peak_index] - # calcualte grandients starting from the max peak and going to the left to find the left margin of the window + # calcualte grandients starting from the max peak and going to the left to + # find the left margin of the window yi_left = yi[:maxposition] yi_grad_left = np.gradient(yi_left[::-1], 0.9) change_grad_pos_left = ( np.where(yi_grad_left[:-1] * yi_grad_left[1:] < 0)[0] + 1 )[0] - # calcualte grandients starting from the max peak and going to the right to find the right margin of the window + # calcualte grandients starting from the max peak and going to the right to + # find the right margin of the window yi_right = yi[maxposition:] yi_grad_right = np.gradient(yi_right, 0.5) change_grad_pos_right = ( diff --git a/src/nectarchain/makers/extractor/utils.py b/src/nectarchain/makers/extractor/utils.py index 0fc1804a..4682d216 100644 --- a/src/nectarchain/makers/extractor/utils.py +++ b/src/nectarchain/makers/extractor/utils.py @@ -1,11 +1,11 @@ import logging +from ctapipe.containers import DL1CameraContainer + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from ctapipe.containers import DL1CameraContainer - class CtapipeExtractor: """ @@ -17,7 +17,8 @@ def get_image_peak_time(cameraContainer: DL1CameraContainer): Extracts the image and peak time from a DL1CameraContainer object. Parameters: - cameraContainer (DL1CameraContainer): The DL1CameraContainer object to extract the image and peak time from. + cameraContainer (DL1CameraContainer): The DL1CameraContainer object to extract + the image and peak time from. Returns: tuple: A tuple containing the image and peak time values from the container. diff --git a/src/nectarchain/makers/tests/test_charges_makers.py b/src/nectarchain/makers/tests/test_charges_makers.py index 89deb3ba..2a68b2a0 100644 --- a/src/nectarchain/makers/tests/test_charges_makers.py +++ b/src/nectarchain/makers/tests/test_charges_makers.py @@ -8,7 +8,9 @@ from nectarchain.data.container import ChargesContainer, ChargesContainers from nectarchain.makers import ChargesNectarCAMCalibrationTool -"This test file test the overall workflow of the ChargesNectarCAMCalibrationTool, adapted from the test of the WaveformsNectarCAMCalibrationTool. There are lot of code duplication but I don't care for now. Do you ?" +# This test file test the overall workflow of the ChargesNectarCAMCalibrationTool, +# adapted from the test of the WaveformsNectarCAMCalibrationTool. +# There are lot of code duplication but I don't care for now. Do you ? class TestChargesNectarCAMCalibrationTool: diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core.py index d54dc135..bf6ecf30 100644 --- a/src/nectarchain/makers/tests/test_core.py +++ b/src/nectarchain/makers/tests/test_core.py @@ -71,7 +71,8 @@ def tool_instance_run_file(self): def test_init_output_path(self, tool_instance): expected_path = pathlib.Path( - f"{os.environ.get('NECTARCAMDATA', '/tmp')}/runs/EventsLoopNectarCAMCalibration_run{self.RUN_NUMBER}.h5" + f"{os.environ.get('NECTARCAMDATA', '/tmp')}/runs" + f"/EventsLoopNectarCAMCalibration_run{self.RUN_NUMBER}.h5" ) assert tool_instance.output_path == expected_path assert tool_instance.run_number == self.RUN_NUMBER @@ -408,11 +409,13 @@ def test_write_container_with_triggermap_container( tool_instance_run_file._write_container(container, index_component=0) container.validate.assert_called_once() mock_writer.write.assert_any_call( - table_name=f"{container.containers[EventType.FLATFIELD].__class__.__name__}_0/{EventType.FLATFIELD.name}", + table_name=f"{container.containers[EventType.FLATFIELD].__class__.__name__}\ + _0/{EventType.FLATFIELD.name}", containers=container.containers[EventType.FLATFIELD], ) mock_writer.write.assert_any_call( - table_name=f"{container.containers[EventType.UNKNOWN].__class__.__name__}_0/{EventType.UNKNOWN.name}", + table_name=f"{container.containers[EventType.UNKNOWN].__class__.__name__}\ + _0/{EventType.UNKNOWN.name}", containers=container.containers[EventType.UNKNOWN], ) @@ -425,7 +428,8 @@ def test_write_container_with_invalid_container( container.validate = MagicMock() with pytest.raises( TypeError, - match="component output must be an instance of TriggerMapContainer or NectarCAMContainer", + match="component output must be an instance of\ + TriggerMapContainer or NectarCAMContainer", ): tool_instance_run_file._write_container(container, index_component=0) container.validate.assert_called_once() diff --git a/src/nectarchain/makers/tests/test_waveforms_makers.py b/src/nectarchain/makers/tests/test_waveforms_makers.py index 69fdc375..553db7c7 100644 --- a/src/nectarchain/makers/tests/test_waveforms_makers.py +++ b/src/nectarchain/makers/tests/test_waveforms_makers.py @@ -9,7 +9,11 @@ from nectarchain.data.container import WaveformsContainer, WaveformsContainers from nectarchain.makers import WaveformsNectarCAMCalibrationTool -"This test file test the overall workflow of the WaveformsNectarCAMCalibrationTool, covering also the EventsLoopNectarCAMCalibrationTool, which cannot be tested on data because the ArrayDataComponent within the WaveformsNectarCAMCalibrationTool is an abstract Component. However the EventsLoopNectarCAMCalibrationTool is still covered by unit test using pytest patches to mock the Component behavior." +# This test file test the overall workflow of the WaveformsNectarCAMCalibrationTool, +# covering also the EventsLoopNectarCAMCalibrationTool, which cannot be tested on +# data because the ArrayDataComponent within the WaveformsNectarCAMCalibrationTool +# is an abstract Component. However the EventsLoopNectarCAMCalibrationTool is still +# covered by unit test using pytest patches to mock the Component behavior. class TestWaveformsNectarCAMCalibrationTool: diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py index e7ef2864..20b72def 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py @@ -13,7 +13,8 @@ parser = argparse.ArgumentParser( prog="gain_SPEfit_computation.py", - description=f"compute high and low gain with the Photo-statistic method, output data are saved in $NECTARCAMDATA/../PhotoStat/", + description=f"compute high and low gain with the Photo-statistic\ + method, output data are saved in $NECTARCAMDATA/../PhotoStat/", ) # run numbers parser.add_argument( From d40c122ca6566bb62eab74c3ffd468936d72c06c Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 22:40:27 +0100 Subject: [PATCH 24/36] refactoring folling pep8 --- docs/user-guide/howto-pedestal.rst | 2 +- src/nectarchain/makers/__init__.py | 4 ++-- src/nectarchain/makers/calibration/__init__.py | 4 ++-- .../{flatfieldMakers.py => flatfield_makers.py} | 0 src/nectarchain/makers/calibration/gain/__init__.py | 4 ++-- .../{FlatFieldSPEMakers.py => flatfield_spe_makers.py} | 0 ...WhiteTargetSPEMakers.py => white_target_spe_makers.py} | 0 .../calibration/{pedestalMakers.py => pedestal_makers.py} | 0 .../makers/{chargesMakers.py => charges_makers.py} | 0 src/nectarchain/makers/component/FlatFieldSPEComponent.py | 4 ++-- src/nectarchain/makers/component/PedestalComponent.py | 4 ++-- src/nectarchain/makers/component/__init__.py | 8 ++++---- .../{chargesComponent.py => charges_component.py} | 0 .../component/{gainComponent.py => gain_component.py} | 0 .../makers/component/photostatistic_component.py | 4 ++-- src/nectarchain/makers/component/spe/spe_algorithm.py | 2 +- .../{waveformsComponent.py => waveforms_component.py} | 0 .../makers/{waveformsMakers.py => waveforms_makers.py} | 0 18 files changed, 18 insertions(+), 18 deletions(-) rename src/nectarchain/makers/calibration/{flatfieldMakers.py => flatfield_makers.py} (100%) rename src/nectarchain/makers/calibration/gain/{FlatFieldSPEMakers.py => flatfield_spe_makers.py} (100%) rename src/nectarchain/makers/calibration/gain/{WhiteTargetSPEMakers.py => white_target_spe_makers.py} (100%) rename src/nectarchain/makers/calibration/{pedestalMakers.py => pedestal_makers.py} (100%) rename src/nectarchain/makers/{chargesMakers.py => charges_makers.py} (100%) rename src/nectarchain/makers/component/{chargesComponent.py => charges_component.py} (100%) rename src/nectarchain/makers/component/{gainComponent.py => gain_component.py} (100%) rename src/nectarchain/makers/component/{waveformsComponent.py => waveforms_component.py} (100%) rename src/nectarchain/makers/{waveformsMakers.py => waveforms_makers.py} (100%) diff --git a/docs/user-guide/howto-pedestal.rst b/docs/user-guide/howto-pedestal.rst index d0c44285..cb6c397e 100644 --- a/docs/user-guide/howto-pedestal.rst +++ b/docs/user-guide/howto-pedestal.rst @@ -25,7 +25,7 @@ processed in slices with a fixed number of events set by the The pedestal estimation tool inherits the configurable parameters of the -`~nectarchain.makers.component.PedestalComponent.PedestalEstimationComponent`. +`~nectarchain.makers.component.pedestal_component.PedestalEstimationComponent`. The data can be filtered based on time using the ``ucts_tmin`` and ``ucts_tmax`` parameters and to eliminate outlier waveforms using the ``filter_method`` parameter. Two different methods to exclude outlier waveforms are implemented: diff --git a/src/nectarchain/makers/__init__.py b/src/nectarchain/makers/__init__.py index 1c55395a..dcca88f6 100644 --- a/src/nectarchain/makers/__init__.py +++ b/src/nectarchain/makers/__init__.py @@ -2,12 +2,12 @@ the makers folder. """ -from .chargesMakers import ChargesNectarCAMCalibrationTool +from .charges_makers import ChargesNectarCAMCalibrationTool from .core import ( DelimiterLoopNectarCAMCalibrationTool, EventsLoopNectarCAMCalibrationTool, ) -from .waveformsMakers import WaveformsNectarCAMCalibrationTool +from .waveforms_makers import WaveformsNectarCAMCalibrationTool __all__ = [ "ChargesNectarCAMCalibrationTool", diff --git a/src/nectarchain/makers/calibration/__init__.py b/src/nectarchain/makers/calibration/__init__.py index 92bc32c7..abc33bd6 100644 --- a/src/nectarchain/makers/calibration/__init__.py +++ b/src/nectarchain/makers/calibration/__init__.py @@ -1,4 +1,4 @@ -from .flatfieldMakers import FlatfieldNectarCAMCalibrationTool +from .flatfield_makers import FlatfieldNectarCAMCalibrationTool from .gain import ( FlatFieldSPECombinedStdNectarCAMCalibrationTool, FlatFieldSPEHHVNectarCAMCalibrationTool, @@ -7,7 +7,7 @@ FlatFieldSPENominalStdNectarCAMCalibrationTool, PhotoStatisticNectarCAMCalibrationTool, ) -from .pedestalMakers import PedestalNectarCAMCalibrationTool +from .pedestal_makers import PedestalNectarCAMCalibrationTool __all__ = [ "FlatfieldNectarCAMCalibrationTool", diff --git a/src/nectarchain/makers/calibration/flatfieldMakers.py b/src/nectarchain/makers/calibration/flatfield_makers.py similarity index 100% rename from src/nectarchain/makers/calibration/flatfieldMakers.py rename to src/nectarchain/makers/calibration/flatfield_makers.py diff --git a/src/nectarchain/makers/calibration/gain/__init__.py b/src/nectarchain/makers/calibration/gain/__init__.py index 44498cb4..eaa6cc9e 100644 --- a/src/nectarchain/makers/calibration/gain/__init__.py +++ b/src/nectarchain/makers/calibration/gain/__init__.py @@ -1,4 +1,4 @@ -from .FlatFieldSPEMakers import ( +from .flatfield_spe_makers import ( FlatFieldSPECombinedStdNectarCAMCalibrationTool, FlatFieldSPEHHVNectarCAMCalibrationTool, FlatFieldSPEHHVStdNectarCAMCalibrationTool, @@ -7,7 +7,7 @@ ) from .photostat_makers import PhotoStatisticNectarCAMCalibrationTool -# from .WhiteTargetSPEMakers import * +# from .white_target_spe_makers import * __all__ = [ "FlatFieldSPENominalNectarCAMCalibrationTool", diff --git a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/flatfield_spe_makers.py similarity index 100% rename from src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py rename to src/nectarchain/makers/calibration/gain/flatfield_spe_makers.py diff --git a/src/nectarchain/makers/calibration/gain/WhiteTargetSPEMakers.py b/src/nectarchain/makers/calibration/gain/white_target_spe_makers.py similarity index 100% rename from src/nectarchain/makers/calibration/gain/WhiteTargetSPEMakers.py rename to src/nectarchain/makers/calibration/gain/white_target_spe_makers.py diff --git a/src/nectarchain/makers/calibration/pedestalMakers.py b/src/nectarchain/makers/calibration/pedestal_makers.py similarity index 100% rename from src/nectarchain/makers/calibration/pedestalMakers.py rename to src/nectarchain/makers/calibration/pedestal_makers.py diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/charges_makers.py similarity index 100% rename from src/nectarchain/makers/chargesMakers.py rename to src/nectarchain/makers/charges_makers.py diff --git a/src/nectarchain/makers/component/FlatFieldSPEComponent.py b/src/nectarchain/makers/component/FlatFieldSPEComponent.py index f128eeb3..f8a75088 100644 --- a/src/nectarchain/makers/component/FlatFieldSPEComponent.py +++ b/src/nectarchain/makers/component/FlatFieldSPEComponent.py @@ -7,8 +7,8 @@ from ...data.container import merge_map_ArrayDataContainer from ...utils import ComponentUtils -from .chargesComponent import ChargesComponent -from .gainComponent import GainNectarCAMComponent +from .charges_component import ChargesComponent +from .gain_component import GainNectarCAMComponent from .spe import SPECombinedalgorithm # noqa from .spe import SPEHHValgorithm # noqa from .spe import SPEHHVStdalgorithm # noqa diff --git a/src/nectarchain/makers/component/PedestalComponent.py b/src/nectarchain/makers/component/PedestalComponent.py index ad255c78..6900cae3 100644 --- a/src/nectarchain/makers/component/PedestalComponent.py +++ b/src/nectarchain/makers/component/PedestalComponent.py @@ -10,9 +10,9 @@ from ...data.container import NectarCAMPedestalContainer, PedestalFlagBits from ...utils import ComponentUtils -from .chargesComponent import ChargesComponent +from .charges_component import ChargesComponent from .core import NectarCAMComponent -from .waveformsComponent import WaveformsComponent +from .waveforms_component import WaveformsComponent logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) diff --git a/src/nectarchain/makers/component/__init__.py b/src/nectarchain/makers/component/__init__.py index 880c803d..73f131fa 100644 --- a/src/nectarchain/makers/component/__init__.py +++ b/src/nectarchain/makers/component/__init__.py @@ -1,14 +1,14 @@ -from .chargesComponent import ChargesComponent +from .charges_component import ChargesComponent from .core import ArrayDataComponent, NectarCAMComponent, get_valid_component -from .FlatFieldSPEComponent import ( +from .flatfield_spe_component import ( FlatFieldCombinedSPEStdNectarCAMComponent, FlatFieldSingleHHVSPENectarCAMComponent, FlatFieldSingleHHVSPEStdNectarCAMComponent, FlatFieldSingleNominalSPENectarCAMComponent, FlatFieldSingleNominalSPEStdNectarCAMComponent, ) -from .gainComponent import GainNectarCAMComponent -from .PedestalComponent import PedestalEstimationComponent +from .gain_component import GainNectarCAMComponent +from .pedestal_component import PedestalEstimationComponent from .photostatistic_algorithm import PhotoStatisticAlgorithm from .photostatistic_component import PhotoStatisticNectarCAMComponent from .spe import SPECombinedalgorithm, SPEHHValgorithm, SPEHHVStdalgorithm diff --git a/src/nectarchain/makers/component/chargesComponent.py b/src/nectarchain/makers/component/charges_component.py similarity index 100% rename from src/nectarchain/makers/component/chargesComponent.py rename to src/nectarchain/makers/component/charges_component.py diff --git a/src/nectarchain/makers/component/gainComponent.py b/src/nectarchain/makers/component/gain_component.py similarity index 100% rename from src/nectarchain/makers/component/gainComponent.py rename to src/nectarchain/makers/component/gain_component.py diff --git a/src/nectarchain/makers/component/photostatistic_component.py b/src/nectarchain/makers/component/photostatistic_component.py index 0e3c9d79..3facaa2b 100644 --- a/src/nectarchain/makers/component/photostatistic_component.py +++ b/src/nectarchain/makers/component/photostatistic_component.py @@ -8,8 +8,8 @@ from ...data.container import SPEfitContainer, merge_map_ArrayDataContainer from ...utils import ComponentUtils -from .chargesComponent import ChargesComponent -from .gainComponent import GainNectarCAMComponent +from .charges_component import ChargesComponent +from .gain_component import GainNectarCAMComponent from .photostatistic_algorithm import PhotoStatisticAlgorithm # noqa logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") diff --git a/src/nectarchain/makers/component/spe/spe_algorithm.py b/src/nectarchain/makers/component/spe/spe_algorithm.py index 46ed5d90..03eda6a8 100644 --- a/src/nectarchain/makers/component/spe/spe_algorithm.py +++ b/src/nectarchain/makers/component/spe/spe_algorithm.py @@ -25,7 +25,7 @@ from ....data.container import ChargesContainer, SPEfitContainer from ....utils import MPE2, MeanValueError, Statistics, UtilsMinuit, weight_gaussian -from ..chargesComponent import ChargesComponent +from ..charges_component import ChargesComponent from .parameters import Parameter, Parameters mplstyle.use("fast") diff --git a/src/nectarchain/makers/component/waveformsComponent.py b/src/nectarchain/makers/component/waveforms_component.py similarity index 100% rename from src/nectarchain/makers/component/waveformsComponent.py rename to src/nectarchain/makers/component/waveforms_component.py diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveforms_makers.py similarity index 100% rename from src/nectarchain/makers/waveformsMakers.py rename to src/nectarchain/makers/waveforms_makers.py From 65c196d9b3b9b894b5dadfca9403cda4d8d221f2 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 22:41:23 +0100 Subject: [PATCH 25/36] folling refactring --- .../{FlatFieldSPEComponent.py => flatfield_spe_component.py} | 0 .../component/{PedestalComponent.py => pedestal_component.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/nectarchain/makers/component/{FlatFieldSPEComponent.py => flatfield_spe_component.py} (100%) rename src/nectarchain/makers/component/{PedestalComponent.py => pedestal_component.py} (100%) diff --git a/src/nectarchain/makers/component/FlatFieldSPEComponent.py b/src/nectarchain/makers/component/flatfield_spe_component.py similarity index 100% rename from src/nectarchain/makers/component/FlatFieldSPEComponent.py rename to src/nectarchain/makers/component/flatfield_spe_component.py diff --git a/src/nectarchain/makers/component/PedestalComponent.py b/src/nectarchain/makers/component/pedestal_component.py similarity index 100% rename from src/nectarchain/makers/component/PedestalComponent.py rename to src/nectarchain/makers/component/pedestal_component.py From 24d8ce4eb4b3cb0d4a65af8c76ccd73033623c03 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 22:51:42 +0100 Subject: [PATCH 26/36] dqm flake8 --- src/nectarchain/dqm/charge_integration.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index 4e0aacae..a7e4a786 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -1,15 +1,13 @@ import ctapipe.instrument.camera.readout import numpy as np from ctapipe.coordinates import EngineeringCameraFrame -from ctapipe.image.extractor import ( - FixedWindowSum, - FullWaveformSum, - GlobalPeakWindowSum, - LocalPeakWindowSum, - NeighborPeakWindowSum, - SlidingWindowMaxSum, - TwoPassWindowSum, -) +from ctapipe.image.extractor import FixedWindowSum # noqa +from ctapipe.image.extractor import FullWaveformSum # noqa +from ctapipe.image.extractor import GlobalPeakWindowSum # noqa +from ctapipe.image.extractor import LocalPeakWindowSum # noqa +from ctapipe.image.extractor import NeighborPeakWindowSum # noqa +from ctapipe.image.extractor import SlidingWindowMaxSum # noqa +from ctapipe.image.extractor import TwoPassWindowSum # noqa from ctapipe.visualization import CameraDisplay from ctapipe_io_nectarcam import constants from matplotlib import pyplot as plt From d571cb358b8a6d232e742dda61367474418f5d63 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 23:05:18 +0100 Subject: [PATCH 27/36] refactoring containers --- docs/user-guide/env-vars.rst | 2 +- docs/user-guide/howto-pedestal.rst | 4 ++-- src/nectarchain/data/container/__init__.py | 8 ++++---- .../{chargesContainer.py => charges_container.py} | 0 .../container/{gainContainer.py => gain_container.py} | 0 .../{pedestalContainer.py => pedestal_container.py} | 0 .../{waveformsContainer.py => waveforms_container.py} | 0 src/nectarchain/makers/component/__init__.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) rename src/nectarchain/data/container/{chargesContainer.py => charges_container.py} (100%) rename src/nectarchain/data/container/{gainContainer.py => gain_container.py} (100%) rename src/nectarchain/data/container/{pedestalContainer.py => pedestal_container.py} (100%) rename src/nectarchain/data/container/{waveformsContainer.py => waveforms_container.py} (100%) diff --git a/docs/user-guide/env-vars.rst b/docs/user-guide/env-vars.rst index 2e9ffa00..1d2c344b 100644 --- a/docs/user-guide/env-vars.rst +++ b/docs/user-guide/env-vars.rst @@ -9,7 +9,7 @@ However, some environment variables are needed for ``nectarchain`` to work prope Mandatory --------- -:``NECTARCAMDATA``: path to local NectarCAM data. It can contain ``fits.fz`` run files, `~nectarchain.data.container.waveformsContainer.WaveformsContainer` or `~nectarchain.data.container.chargesContainer.ChargesContainer` HDF5 files. This is also where the `~nectarchain.data.management.DataManagement.findrun` method will automatically store NectarCAM run files when fetched from DIRAC. +:``NECTARCAMDATA``: path to local NectarCAM data. It can contain ``fits.fz`` run files, `~nectarchain.data.container.waveforms_container.WaveformsContainer` or `~nectarchain.data.container.charges_container.ChargesContainer` HDF5 files. This is also where the `~nectarchain.data.management.DataManagement.findrun` method will automatically store NectarCAM run files when fetched from DIRAC. Optional -------- diff --git a/docs/user-guide/howto-pedestal.rst b/docs/user-guide/howto-pedestal.rst index cb6c397e..e1eaa50d 100644 --- a/docs/user-guide/howto-pedestal.rst +++ b/docs/user-guide/howto-pedestal.rst @@ -46,11 +46,11 @@ To run the example script: Inspect the results ========================= The results are stored in a -`~nectarchain.data.container.pedestalContainer.NectarCAMPedestalContainer`. The +`~nectarchain.data.container.pedestal_container.NectarCAMPedestalContainer`. The results include information on pixels that were flagged as having an abnormal behavior during the computation of the pedestals. The flags are defined in in -`~nectarchain.data.container.pedestalContainer.PedestalFlagBits`. The +`~nectarchain.data.container.pedestal_container.PedestalFlagBits`. The results are accessible on the fly if the tool is run interactively (as in the example above) and stored in a `.h5` file. The user script `nectarchain/user_scripts/ltibaldo/show_pedestal_output.py` provides an example of how to access the results from disk and produce some plots: diff --git a/src/nectarchain/data/container/__init__.py b/src/nectarchain/data/container/__init__.py index 3a969d84..26147bbd 100644 --- a/src/nectarchain/data/container/__init__.py +++ b/src/nectarchain/data/container/__init__.py @@ -1,7 +1,7 @@ """This file is used to import all the containerclasses in the data/container folder.""" -from .chargesContainer import ChargesContainer, ChargesContainers +from .charges_container import ChargesContainer, ChargesContainers from .core import ( ArrayDataContainer, NectarCAMContainer, @@ -9,9 +9,9 @@ get_array_keys, merge_map_ArrayDataContainer, ) -from .gainContainer import GainContainer, SPEfitContainer -from .pedestalContainer import NectarCAMPedestalContainer, PedestalFlagBits -from .waveformsContainer import WaveformsContainer, WaveformsContainers +from .gain_container import GainContainer, SPEfitContainer +from .pedestal_container import NectarCAMPedestalContainer, PedestalFlagBits +from .waveforms_container import WaveformsContainer, WaveformsContainers __all__ = [ "ArrayDataContainer", diff --git a/src/nectarchain/data/container/chargesContainer.py b/src/nectarchain/data/container/charges_container.py similarity index 100% rename from src/nectarchain/data/container/chargesContainer.py rename to src/nectarchain/data/container/charges_container.py diff --git a/src/nectarchain/data/container/gainContainer.py b/src/nectarchain/data/container/gain_container.py similarity index 100% rename from src/nectarchain/data/container/gainContainer.py rename to src/nectarchain/data/container/gain_container.py diff --git a/src/nectarchain/data/container/pedestalContainer.py b/src/nectarchain/data/container/pedestal_container.py similarity index 100% rename from src/nectarchain/data/container/pedestalContainer.py rename to src/nectarchain/data/container/pedestal_container.py diff --git a/src/nectarchain/data/container/waveformsContainer.py b/src/nectarchain/data/container/waveforms_container.py similarity index 100% rename from src/nectarchain/data/container/waveformsContainer.py rename to src/nectarchain/data/container/waveforms_container.py diff --git a/src/nectarchain/makers/component/__init__.py b/src/nectarchain/makers/component/__init__.py index 73f131fa..62aae577 100644 --- a/src/nectarchain/makers/component/__init__.py +++ b/src/nectarchain/makers/component/__init__.py @@ -12,7 +12,7 @@ from .photostatistic_algorithm import PhotoStatisticAlgorithm from .photostatistic_component import PhotoStatisticNectarCAMComponent from .spe import SPECombinedalgorithm, SPEHHValgorithm, SPEHHVStdalgorithm -from .waveformsComponent import WaveformsComponent +from .waveforms_component import WaveformsComponent __all__ = [ "ArrayDataComponent", From 9b9a63c7b7610d0789ebbcea43a770879533093d Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 23:27:48 +0100 Subject: [PATCH 28/36] fix bug formatting --- src/nectarchain/data/container/core.py | 2 +- src/nectarchain/makers/tests/test_core.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index 8e783f10..15a71d28 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -258,7 +258,7 @@ def _container_from_hdf5( """ if isinstance(path, str): path = Path(path) - _ = importlib.import_module(f"{container_class.__module__}") + module = importlib.import_module(f"{container_class.__module__}") # noqa container = eval(f"module.{container_class.__name__}")() with HDF5TableReader(path) as reader: diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core.py index bf6ecf30..7307bedb 100644 --- a/src/nectarchain/makers/tests/test_core.py +++ b/src/nectarchain/makers/tests/test_core.py @@ -409,13 +409,13 @@ def test_write_container_with_triggermap_container( tool_instance_run_file._write_container(container, index_component=0) container.validate.assert_called_once() mock_writer.write.assert_any_call( - table_name=f"{container.containers[EventType.FLATFIELD].__class__.__name__}\ - _0/{EventType.FLATFIELD.name}", + table_name=f"{container.containers[EventType.FLATFIELD].__class__.__name__}" + f"_0/{EventType.FLATFIELD.name}", containers=container.containers[EventType.FLATFIELD], ) mock_writer.write.assert_any_call( - table_name=f"{container.containers[EventType.UNKNOWN].__class__.__name__}\ - _0/{EventType.UNKNOWN.name}", + table_name=f"{container.containers[EventType.UNKNOWN].__class__.__name__}" + f"_0/{EventType.UNKNOWN.name}", containers=container.containers[EventType.UNKNOWN], ) From 532f961d7ef29d94d35b06e7837ef0cacb2f6702 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 8 Jan 2025 23:54:05 +0100 Subject: [PATCH 29/36] shitty python formatting --- src/nectarchain/data/container/core.py | 30 +++++++-------- .../calibration/gain/flatfield_spe_makers.py | 32 ++++++++++------ .../calibration/gain/photostat_makers.py | 37 +++++++++++-------- src/nectarchain/makers/charges_makers.py | 16 ++++---- .../component/photostatistic_component.py | 4 +- .../makers/component/spe/parameters.py | 8 ++-- 6 files changed, 71 insertions(+), 56 deletions(-) diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index 15a71d28..a1e010d6 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -264,9 +264,9 @@ def _container_from_hdf5( with HDF5TableReader(path) as reader: if len(reader._h5file.root.__members__) > 1 and slice_index is None: log.info( - f"reading {container_class.__name__} containing\ - {len(reader._h5file.root.__members__)}\ - slices, will return a generator" + f"reading {container_class.__name__} containing" + f"{len(reader._h5file.root.__members__)}" + f"slices, will return a generator" ) for data in reader._h5file.root.__members__: # container.containers[data] = @@ -286,8 +286,8 @@ def _container_from_hdf5( _waveforms_data = np.array(waveforms_data)[_mask] if len(_waveforms_data) == 1: tableReader = reader.read( - table_name=f"/{data}/\ - {_waveforms_data[0]}/{trigger.name}", + table_name=f"/{data}/{_waveforms_data[0]}" + f"/{trigger.name}", containers=_container, ) # container.containers[data].containers[trigger] = @@ -296,9 +296,9 @@ def _container_from_hdf5( else: log.info( - f"there is {len(_waveforms_data)} entry\ - corresponding to a {container_class}\ - table save, unable to load" + f"there is {len(_waveforms_data)} entry" + f"corresponding to a {container_class}" + f"table save, unable to load" ) except NoSuchNodeError as err: log.warning(err) @@ -309,15 +309,15 @@ def _container_from_hdf5( else: if slice_index is None: log.info( - f"reading {container_class.__name__} containing\ - a single slice,\ - will return the {container_class.__name__} instance" + f"reading {container_class.__name__} containing" + f"a single slice," + f"will return the {container_class.__name__} instance" ) data = "data" else: log.info( - f"reading slice {slice_index} of {container_class.__name__},\ - will return the {container_class.__name__} instance" + f"reading slice {slice_index} of {container_class.__name__}," + f"will return the {container_class.__name__} instance" ) data = f"data_{slice_index}" for key, trigger in EventType.__members__.items(): @@ -326,8 +326,8 @@ def _container_from_hdf5( f"module.{container.fields['containers'].default_factory.args[0].__name__}" # noqa ) tableReader = reader.read( - table_name=f"/{data}/{_container.__name__}_\ - {index_component}/{trigger.name}", + table_name=f"/{data}/{_container.__name__}_" + f"{index_component}/{trigger.name}", containers=_container, ) container.containers[trigger] = next(tableReader) diff --git a/src/nectarchain/makers/calibration/gain/flatfield_spe_makers.py b/src/nectarchain/makers/calibration/gain/flatfield_spe_makers.py index 96654928..07854746 100644 --- a/src/nectarchain/makers/calibration/gain/flatfield_spe_makers.py +++ b/src/nectarchain/makers/calibration/gain/flatfield_spe_makers.py @@ -71,11 +71,15 @@ def _init_output_path(self): else: ext = f"_sliced{self.events_per_slice}.h5" if self.max_events is None: - filename = f"{self.name}_run{self.run_number}_\ - {self.method}_{str_extractor_kwargs}{ext}" + filename = ( + f"{self.name}_run{self.run_number}_" + f"{self.method}_{str_extractor_kwargs}{ext}" + ) else: - filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}\ - _{self.method}_{str_extractor_kwargs}{ext}" + filename = ( + f"{self.name}_run{self.run_number}_maxevents" + f"{self.max_events}_{self.method}_{str_extractor_kwargs}{ext}" + ) self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/SPEfit/{filename}" @@ -101,10 +105,10 @@ def start( if self.reload_events or len(files) != 1: if len(files) != 1: self.log.info( - f"{len(files)} computed charges files found with max_events >\ - {self.max_events} for run {self.run_number} with extraction\ - method {self.method} and {str_extractor_kwargs},\n reload\ - charges from event loop" + f"{len(files)} computed charges files found with max_events >" + f"{self.max_events} for run {self.run_number} with extraction" + f"method {self.method} and {str_extractor_kwargs},\n reload" + f"charges from event loop" ) super().start( n_events=n_events, @@ -208,11 +212,15 @@ def _init_output_path(self): self.extractor_kwargs ) if self.max_events is None: - filename = f"{self.name}_run{self.run_number}_HHV{HHVrun}_{self.method}_\ - {str_extractor_kwargs}.h5" + filename = ( + f"{self.name}_run{self.run_number}_HHV{HHVrun}_{self.method}_" + f"{str_extractor_kwargs}.h5" + ) else: - filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}_\ - HHV{HHVrun}_{self.method}_{str_extractor_kwargs}.h5" + filename = ( + f"{self.name}_run{self.run_number}_maxevents{self.max_events}_" + f"HHV{HHVrun}_{self.method}_{str_extractor_kwargs}.h5" + ) self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/SPEfit/{filename}" diff --git a/src/nectarchain/makers/calibration/gain/photostat_makers.py b/src/nectarchain/makers/calibration/gain/photostat_makers.py index 48adba17..83d35b3b 100644 --- a/src/nectarchain/makers/calibration/gain/photostat_makers.py +++ b/src/nectarchain/makers/calibration/gain/photostat_makers.py @@ -62,12 +62,17 @@ def _init_output_path(self): self.extractor_kwargs ) if self.max_events is None: - filename = f"{self.name}_FFrun{self.run_number}_{self.method}\ - _{str_extractor_kwargs}_Pedrun{self.Ped_run_number}_FullWaveformSum.h5" + filename = ( + f"{self.name}_FFrun{self.run_number}_{self.method}" + f"_{str_extractor_kwargs}_Pedrun{self.Ped_run_number}" + f"_FullWaveformSum.h5" + ) else: - filename = f"{self.name}_FFrun{self.run_number}_{self.method}\ - _{str_extractor_kwargs}_Pedrun{self.Ped_run_number}_\ - FullWaveformSum_maxevents{self.max_events}.h5" + filename = ( + f"{self.name}_FFrun{self.run_number}_{self.method}" + f"_{str_extractor_kwargs}_Pedrun{self.Ped_run_number}_" + f"FullWaveformSum_maxevents{self.max_events}.h5" + ) self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/PhotoStat/{filename}" ) @@ -106,19 +111,19 @@ def start( if self.reload_events or len(FF_files) != 1 or len(Ped_files) != 1: if len(FF_files) != 1 or len(Ped_files) != 1: self.log.info( - f"{len(FF_files)} computed charges FF files found\ - with max_events >\ - {self.max_events} for run {self.run_number}\ - with extraction method\ - {self.method} and {str_extractor_kwargs},\n reload charges\ - from event loop" + f"{len(FF_files)} computed charges FF files found" + f"with max_events >" + f"{self.max_events} for run {self.run_number}" + f"with extraction method" + f"{self.method} and {str_extractor_kwargs},\n reload charges" + f"from event loop" ) self.log.info( - f"{len(Ped_files)} computed charges FF files found\ - with max_events >\ - {self.max_events} for run {self.Ped_run_number}\ - with extraction\ - method FullWaveformSum,\n reload charges from event loop" + f"{len(Ped_files)} computed charges FF files found" + f"with max_events >" + f"{self.max_events} for run {self.Ped_run_number}" + f"with extraction" + f"method FullWaveformSum,\n reload charges from event loop" ) super().start( diff --git a/src/nectarchain/makers/charges_makers.py b/src/nectarchain/makers/charges_makers.py index 0fbcc2eb..fe5905f4 100644 --- a/src/nectarchain/makers/charges_makers.py +++ b/src/nectarchain/makers/charges_makers.py @@ -76,9 +76,9 @@ def start( ) if len(files) != 1: self.log.info( - f"{len(files)} computed wavforms files found with max_events >=\ - {self.max_events} for run {self.run_number}, reload waveforms\ - from event loop" + f"{len(files)} computed wavforms files found with max_events >=" + f"{self.max_events} for run {self.run_number}, reload waveforms" + f"from event loop" ) super().start( n_events=n_events, @@ -88,9 +88,9 @@ def start( ) else: self.log.info( - f"{files[0]} is the computed wavforms files found\ - with max_events >=\ - {self.max_events} for run {self.run_number}" + f"{files[0]} is the computed wavforms files found" + f"with max_events >=" + f"{self.max_events} for run {self.run_number}" ) waveformsContainers = WaveformsContainers.from_hdf5(files[0]) if not (isinstance(waveformsContainers, WaveformsContainer)): @@ -115,8 +115,8 @@ def start( self._write_container(container=chargesContainers) else: self.log.debug( - f"WaveformsContainer file contains {n_slices} slices of the\ - run events" + f"WaveformsContainer file contains {n_slices} slices of the" + f"run events" ) for slice_index, _waveformsContainers in enumerate( waveformsContainers diff --git a/src/nectarchain/makers/component/photostatistic_component.py b/src/nectarchain/makers/component/photostatistic_component.py index 3facaa2b..8c3982a1 100644 --- a/src/nectarchain/makers/component/photostatistic_component.py +++ b/src/nectarchain/makers/component/photostatistic_component.py @@ -104,8 +104,8 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.Ped_chargesComponent(event=event, *args, **kwargs) else: self.log.warning( - f"event {event.index.event_id} is event type {event.trigger.event_type}\ - which is not used here" + f"event {event.index.event_id} is event type {event.trigger.event_type}" + f"which is not used here" ) def finish(self, *args, **kwargs): diff --git a/src/nectarchain/makers/component/spe/parameters.py b/src/nectarchain/makers/component/spe/parameters.py index 26db7aae..65280b3a 100644 --- a/src/nectarchain/makers/component/spe/parameters.py +++ b/src/nectarchain/makers/component/spe/parameters.py @@ -42,9 +42,11 @@ def from_instance(cls, parameter): ) def __str__(self): - return f"name : {self.__name}, value : {self.__value}, error : {self.__error},\ - unit : {self.__unit}, min : {self.__min}, max : {self.__max},\ - frozen : {self.__frozen}" + return ( + f"name : {self.__name}, value : {self.__value}, error : {self.__error}," + f"unit : {self.__unit}, min : {self.__min}, max : {self.__max}," + f"frozen : {self.__frozen}" + ) @property def name(self): From f6cf9720790e11482278cab0e6164010738c8cd6 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 9 Jan 2025 00:24:09 +0100 Subject: [PATCH 30/36] test bugfix --- src/nectarchain/makers/tests/test_core.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core.py index 7307bedb..d447ba8c 100644 --- a/src/nectarchain/makers/tests/test_core.py +++ b/src/nectarchain/makers/tests/test_core.py @@ -58,15 +58,15 @@ class TestEventsLoopNectarCAMCalibrationTool(TestBaseNectarCAMCalibrationTool): @pytest.fixture def tool_instance(self): - return EventsLoopNectarCAMCalibrationTool( - run_number=self.RUN_NUMBER, - ) + return EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER) @pytest.fixture def tool_instance_run_file(self): return EventsLoopNectarCAMCalibrationTool( run_number=self.RUN_NUMBER, run_file=self.RUN_FILE, + output_path=pathlib.Path(f"/tmp/{np.random.random()}test_output.h5") + # to avoid I/O conflicts between tests ) def test_init_output_path(self, tool_instance): @@ -289,12 +289,16 @@ def test_split_run(self, tool_instance): assert not (tool_instance.split_run(n_events_in_slice=2, event=event)) @patch("nectarchain.makers.core.Component") - def test_start(self, mock_component, tool_instance_run_file): + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) + def test_start(self, mock_finish_component, mock_component, tool_instance_run_file): tool_instance_run_file.overwrite = True tool_instance_run_file.setup() n_events = len(tool_instance_run_file.event_source) tool_instance_run_file.components = [mock_component.from_name.return_value] tool_instance_run_file.start() + tool_instance_run_file.finish() assert tool_instance_run_file._n_traited_events == n_events @patch("nectarchain.makers.core.Component") @@ -331,7 +335,10 @@ def test_start_sliced( n_events = len(tool_instance_run_file.event_source) tool_instance_run_file.components = [MockComponent()] tool_instance_run_file.start() - assert mock_finish_components.call_count == n_events // self.EVENTS_PER_SLICE + tool_instance_run_file.finish() + assert ( + mock_finish_components.call_count == n_events // self.EVENTS_PER_SLICE + 1 + ) assert mock_setup_components.call_count == n_events // self.EVENTS_PER_SLICE + 1 @patch("nectarchain.makers.core.Component") @@ -428,8 +435,8 @@ def test_write_container_with_invalid_container( container.validate = MagicMock() with pytest.raises( TypeError, - match="component output must be an instance of\ - TriggerMapContainer or NectarCAMContainer", + match="component output must be an instance " + "of TriggerMapContainer or NectarCAMContainer", ): tool_instance_run_file._write_container(container, index_component=0) container.validate.assert_called_once() From f950359d0498910574e084d8f4f1a0113f1a0203 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 9 Jan 2025 02:30:55 +0100 Subject: [PATCH 31/36] pedestal maker : implementation of I/O with containers --- src/nectarchain/data/container/__init__.py | 7 +- src/nectarchain/data/container/core.py | 119 +++++++++------- .../data/container/pedestal_container.py | 26 +++- .../makers/calibration/pedestal_makers.py | 133 +++++++++--------- .../calibration/tests/test_pedestal_tool.py | 127 +++++++++-------- 5 files changed, 235 insertions(+), 177 deletions(-) diff --git a/src/nectarchain/data/container/__init__.py b/src/nectarchain/data/container/__init__.py index 26147bbd..574b5ed1 100644 --- a/src/nectarchain/data/container/__init__.py +++ b/src/nectarchain/data/container/__init__.py @@ -10,7 +10,11 @@ merge_map_ArrayDataContainer, ) from .gain_container import GainContainer, SPEfitContainer -from .pedestal_container import NectarCAMPedestalContainer, PedestalFlagBits +from .pedestal_container import ( + NectarCAMPedestalContainer, + NectarCAMPedestalContainers, + PedestalFlagBits, +) from .waveforms_container import WaveformsContainer, WaveformsContainers __all__ = [ @@ -26,5 +30,6 @@ "GainContainer", "SPEfitContainer", "NectarCAMPedestalContainer", + "NectarCAMPedestalContainers", "PedestalFlagBits", ] diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index a1e010d6..726f6e49 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -226,6 +226,9 @@ def from_hdf5(cls, path, slice_index=None, index_component=0): def _container_from_hdf5( path, container_class, slice_index=None, index_component=0 ): + # The way this method is coded is bad, there are confliuct behavior bettween + # containers inherited from TriggerMapContainer to truly be mapped with trigger, + # and those mapped with slices """Reads a container from an HDF5 file. Parameters: @@ -268,44 +271,48 @@ def _container_from_hdf5( f"{len(reader._h5file.root.__members__)}" f"slices, will return a generator" ) - for data in reader._h5file.root.__members__: + for data in np.sort(reader._h5file.root.__members__): # container.containers[data] = # eval(f"module.{container_class.__name__}s")() - for key, trigger in EventType.__members__.items(): - try: - _container = eval( - f"module." - f"{container.fields['containers'].default_factory.args[0].__name__}" # noqa - ) - waveforms_data = eval( - f"reader._h5file.root.{data}.__members__" + + _container = eval( + f"module." + f"{container.fields['containers'].default_factory.args[0].__name__}" # noqa + ) + waveforms_data = eval(f"reader._h5file.root.{data}.__members__") + _mask = [_container.__name__ in _word for _word in waveforms_data] + _waveforms_data = np.array(waveforms_data)[_mask] + if len(_waveforms_data) == 1: + if issubclass(_container, TriggerMapContainer) or issubclass( + _container, ArrayDataContainer + ): + for key, trigger in EventType.__members__.items(): + try: + tableReader = reader.read( + table_name=f"/{data}/{_waveforms_data[0]}" + f"/{trigger.name}", + containers=_container, + ) + container.containers[trigger] = next(tableReader) + except NoSuchNodeError as err: + log.warning(err) + except Exception as err: + log.error(err, exc_info=True) + raise err + else: + tableReader = reader.read( + table_name=f"/{data}/{_waveforms_data[0]}", + containers=_container, ) - _mask = [ - _container.__name__ in _word for _word in waveforms_data - ] - _waveforms_data = np.array(waveforms_data)[_mask] - if len(_waveforms_data) == 1: - tableReader = reader.read( - table_name=f"/{data}/{_waveforms_data[0]}" - f"/{trigger.name}", - containers=_container, - ) - # container.containers[data].containers[trigger] = - # next(tableReader) - container.containers[trigger] = next(tableReader) - - else: - log.info( - f"there is {len(_waveforms_data)} entry" - f"corresponding to a {container_class}" - f"table save, unable to load" - ) - except NoSuchNodeError as err: - log.warning(err) - except Exception as err: - log.error(err, exc_info=True) - raise err - yield container + container.containers[data] = next(tableReader) + else: + log.info( + f"there is {len(_waveforms_data)} entry" + f"corresponding to a {container_class}" + f"table save, unable to load" + ) + + yield container else: if slice_index is None: log.info( @@ -320,22 +327,32 @@ def _container_from_hdf5( f"will return the {container_class.__name__} instance" ) data = f"data_{slice_index}" - for key, trigger in EventType.__members__.items(): - try: - _container = eval( - f"module.{container.fields['containers'].default_factory.args[0].__name__}" # noqa - ) - tableReader = reader.read( - table_name=f"/{data}/{_container.__name__}_" - f"{index_component}/{trigger.name}", - containers=_container, - ) - container.containers[trigger] = next(tableReader) - except NoSuchNodeError as err: - log.warning(err) - except Exception as err: - log.error(err, exc_info=True) - raise err + _container = eval( + f"module.{container.fields['containers'].default_factory.args[0].__name__}" # noqa + ) + if issubclass(_container, TriggerMapContainer) or issubclass( + _container, ArrayDataContainer + ): + for key, trigger in EventType.__members__.items(): + try: + tableReader = reader.read( + table_name=f"/{data}/{_container.__name__}_" + f"{index_component}/{trigger.name}", + containers=_container, + ) + container.containers[trigger] = next(tableReader) + except NoSuchNodeError as err: + log.warning(err) + except Exception as err: + log.error(err, exc_info=True) + raise err + else: + tableReader = reader.read( + table_name=f"/{data}/{_container.__name__}_" + f"{index_component}", + containers=_container, + ) + container.containers[data] = next(tableReader) yield container def is_empty(self): diff --git a/src/nectarchain/data/container/pedestal_container.py b/src/nectarchain/data/container/pedestal_container.py index 65add6be..b0c11d72 100644 --- a/src/nectarchain/data/container/pedestal_container.py +++ b/src/nectarchain/data/container/pedestal_container.py @@ -2,15 +2,19 @@ from enum import IntFlag, auto, unique import numpy as np -from ctapipe.containers import Field +from ctapipe.containers import Field, Map, partial -from .core import NectarCAMContainer +from .core import NectarCAMContainer, TriggerMapContainer logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -__all__ = ["NectarCAMPedestalContainer", "PedestalFlagBits"] +__all__ = [ + "NectarCAMPedestalContainer", + "PedestalFlagBits", + "NectarCAMPedestalContainers", +] @unique @@ -105,3 +109,19 @@ class NectarCAMPedestalContainer(NectarCAMContainer): The meaning of the mask bits is defined in the class\ ~nectarchain.data.container.PedestalFlagBits", ) + + +class NectarCAMPedestalContainers(TriggerMapContainer): + """Class representing a NectarCAMPedestalContainers.This class inherits from the + `TriggerMapContainer` class and is used to store slices of data + mappings of `NectarCAMPedestalContainer`. + + Attributes: + containers (Field): A field representing the slices + of data mapping of `NectarCAMPedestalContainer`. + """ + + containers = Field( + default_factory=partial(Map, NectarCAMPedestalContainer), + description="slices of data mapping of NectarCAMPedestalContainer", + ) diff --git a/src/nectarchain/makers/calibration/pedestal_makers.py b/src/nectarchain/makers/calibration/pedestal_makers.py index 0db5386f..149111fe 100644 --- a/src/nectarchain/makers/calibration/pedestal_makers.py +++ b/src/nectarchain/makers/calibration/pedestal_makers.py @@ -3,11 +3,10 @@ import pathlib import numpy as np -import tables from ctapipe.core.traits import ComponentNameList from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN, N_GAINS -from ...data.container import NectarCAMPedestalContainer +from ...data.container import NectarCAMPedestalContainer, NectarCAMPedestalContainers from ..component import NectarCAMComponent from .core import NectarCAMCalibrationTool @@ -58,75 +57,75 @@ def _combine_results(self): """ # re-open results - # TODO: update to use nectarchain generators when available - with tables.open_file(self.output_path) as h5file: - # Loop over sliced results to fill the combined results - self.log.info("Combine sliced results") - for i, result in enumerate(h5file.root.__members__): - if result == "data_combined": - log.error( - "Trying to combine results that already contain combined data" - ) - table = h5file.root[result][NectarCAMPedestalContainer.__name__][0] - if i == 0: - # initialize fields for the combined results based on first slice - nsamples = table["nsamples"] - nevents = np.zeros(len(table["nevents"])) - pixels_id = table["pixels_id"] - ucts_timestamp_min = table["ucts_timestamp_min"] - ucts_timestamp_max = table["ucts_timestamp_max"] - pedestal_mean_hg = np.zeros(np.shape(table["pedestal_mean_hg"])) - pedestal_mean_lg = np.zeros(np.shape(table["pedestal_mean_lg"])) - pedestal_std_hg = np.zeros(np.shape(table["pedestal_std_hg"])) - pedestal_std_lg = np.zeros(np.shape(table["pedestal_std_lg"])) - else: - # otherwise consider the overall time interval - ucts_timestamp_min = np.minimum( - ucts_timestamp_min, table["ucts_timestamp_min"] - ) - ucts_timestamp_max = np.maximum( - ucts_timestamp_max, table["ucts_timestamp_max"] - ) - # for all slices - # derive from pixel mask a mask that sets usable pixels - # accept only pixels for which no flags were raised - usable_pixels = table["pixel_mask"] == 0 - # use a pixel only if it has no flag on either channel - usable_pixels = np.logical_and(usable_pixels[0], usable_pixels[1]) - - # cumulated number of events - nevents += table["nevents"] * usable_pixels - - # add mean, std sum elements - pedestal_mean_hg += ( - table["pedestal_mean_hg"] - * table["nevents"][:, np.newaxis] - * usable_pixels[:, np.newaxis] + pedestalContainers = next( + NectarCAMPedestalContainers.from_hdf5(self.output_path) + ) + # Loop over sliced results to fill the combined results + if "data_combined" in pedestalContainers.containers.keys(): + log.error("Trying to combine results that already contain combined data") + self.log.info("Combine sliced results") + for i, (_, pedestalContainer) in enumerate( + pedestalContainers.containers.items() + ): + if i == 0: + # initialize fields for the combined results based on first slice + nsamples = pedestalContainer.nsamples + nevents = np.zeros(len(pedestalContainer.nevents)) + pixels_id = pedestalContainer.pixels_id + ucts_timestamp_min = pedestalContainer.ucts_timestamp_min + ucts_timestamp_max = pedestalContainer.ucts_timestamp_max + pedestal_mean_hg = np.zeros( + np.shape(pedestalContainer.pedestal_mean_hg) ) - pedestal_mean_lg += ( - table["pedestal_mean_lg"] - * table["nevents"][:, np.newaxis] - * usable_pixels[:, np.newaxis] + pedestal_mean_lg = np.zeros( + np.shape(pedestalContainer.pedestal_mean_lg) ) - pedestal_std_hg += ( - table["pedestal_std_hg"] ** 2 - * table["nevents"][:, np.newaxis] - * usable_pixels[:, np.newaxis] + pedestal_std_hg = np.zeros(np.shape(pedestalContainer.pedestal_std_hg)) + pedestal_std_lg = np.zeros(np.shape(pedestalContainer.pedestal_std_lg)) + else: + # otherwise consider the overall time interval + ucts_timestamp_min = np.minimum( + ucts_timestamp_min, pedestalContainer.ucts_timestamp_min ) - pedestal_std_lg += ( - table["pedestal_std_lg"] ** 2 - * table["nevents"][:, np.newaxis] - * usable_pixels[:, np.newaxis] + ucts_timestamp_max = np.maximum( + ucts_timestamp_max, pedestalContainer.ucts_timestamp_max ) - - # calculate final values of mean and std - pedestal_mean_hg /= nevents[:, np.newaxis] - pedestal_mean_lg /= nevents[:, np.newaxis] - pedestal_std_hg /= nevents[:, np.newaxis] - pedestal_std_hg = np.sqrt(pedestal_std_hg) - pedestal_std_lg /= nevents[:, np.newaxis] - pedestal_std_lg = np.sqrt(pedestal_std_lg) - + # for all slices + # derive from pixel mask a mask that sets usable pixels + # accept only pixels for which no flags were raised + usable_pixels = pedestalContainer.pixel_mask == 0 + # use a pixel only if it has no flag on either channel + usable_pixels = np.logical_and(usable_pixels[0], usable_pixels[1]) + # cumulated number of events + nevents += pedestalContainer.nevents * usable_pixels + # add mean, std sum elements + pedestal_mean_hg += ( + pedestalContainer.pedestal_mean_hg + * pedestalContainer.nevents[:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) + pedestal_mean_lg += ( + pedestalContainer.pedestal_mean_lg + * pedestalContainer.nevents[:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) + pedestal_std_hg += ( + pedestalContainer.pedestal_std_hg**2 + * pedestalContainer.nevents[:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) + pedestal_std_lg += ( + pedestalContainer.pedestal_std_lg**2 + * pedestalContainer.nevents[:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) + # calculate final values of mean and std + pedestal_mean_hg /= nevents[:, np.newaxis] + pedestal_mean_lg /= nevents[:, np.newaxis] + pedestal_std_hg /= nevents[:, np.newaxis] + pedestal_std_hg = np.sqrt(pedestal_std_hg) + pedestal_std_lg /= nevents[:, np.newaxis] + pedestal_std_lg = np.sqrt(pedestal_std_lg) # flag bad pixels in overall results based on same criteria as for individual # slides # reconstitute dictionary with cumulated results consistently with diff --git a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py index 9d7f8f6d..af72a269 100644 --- a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py +++ b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py @@ -1,11 +1,10 @@ import tempfile import numpy as np -import tables from ctapipe.utils import get_dataset_path from ctapipe_io_nectarcam.constants import N_SAMPLES -from nectarchain.data.container import NectarCAMPedestalContainer, PedestalFlagBits +from nectarchain.data.container import NectarCAMPedestalContainers, PedestalFlagBits from nectarchain.makers.calibration import PedestalNectarCAMCalibrationTool runs = { @@ -79,57 +78,75 @@ def test_base(self): # Check output on disk # FIXME: use tables for the moment, update when h5 reader in nectarchain # is working - with tables.open_file(outfile) as h5file: - for s in range(n_slices[i]): - # Check individual groups - group_name = "data_{}".format(s + 1) - assert group_name in h5file.root.__members__ - table = h5file.root[group_name][ - NectarCAMPedestalContainer.__name__ - ][0] - assert table["nsamples"] == N_SAMPLES - assert np.allclose(table["nevents"], events_per_slice, atol=7) - assert np.shape(table["pixels_id"]) == (n_pixels,) - assert np.shape(table["pedestal_mean_hg"]) == ( - n_pixels, - N_SAMPLES, - ) - assert np.shape(table["pedestal_mean_lg"]) == ( - n_pixels, - N_SAMPLES, - ) - assert np.shape(table["pedestal_std_hg"]) == ( - n_pixels, - N_SAMPLES, - ) - assert np.shape(table["pedestal_std_lg"]) == ( - n_pixels, - N_SAMPLES, - ) - # Check combined results - group_name = "data_combined" - table = h5file.root[group_name][ - NectarCAMPedestalContainer.__name__ - ][0] - assert table["nsamples"] == N_SAMPLES - assert np.all(table["nevents"] == max_events[i]) - assert np.shape(table["pixels_id"]) == (n_pixels,) - assert table["ucts_timestamp_min"] == np.uint64( - expected_ucts_timestamp_min[i] + pedestalContainers = next( + NectarCAMPedestalContainers.from_hdf5(outfile) + ) + j = 0 + for key, pedestalContainer in pedestalContainers.containers.items(): + if "combined" in key: + continue + # Check individual groups + group_name = "data_{}".format(i + 1) + assert group_name in pedestalContainers.containers.keys() + assert pedestalContainer.nsamples == N_SAMPLES + assert np.allclose( + pedestalContainer.nevents, events_per_slice, atol=7 ) - assert table["ucts_timestamp_max"] == np.uint64( - expected_ucts_timestamp_max[i] + assert np.shape(pedestalContainer.pixels_id) == (n_pixels,) + assert np.shape(pedestalContainer.pedestal_mean_hg) == ( + n_pixels, + N_SAMPLES, ) - assert np.shape(table["pedestal_mean_hg"]) == (n_pixels, N_SAMPLES) - assert np.shape(table["pedestal_mean_lg"]) == (n_pixels, N_SAMPLES) - assert np.shape(table["pedestal_std_hg"]) == (n_pixels, N_SAMPLES) - assert np.shape(table["pedestal_std_lg"]) == (n_pixels, N_SAMPLES) - assert np.allclose(table["pedestal_mean_hg"], 245.0, atol=20.0) - assert np.allclose(table["pedestal_mean_lg"], 245.0, atol=20.0) - assert np.allclose(table["pedestal_std_hg"], 10, atol=10) - assert np.allclose( - table["pedestal_std_lg"], 2.5, atol=2.0 if i == 0 else 2.3 + assert np.shape(pedestalContainer.pedestal_mean_lg) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(pedestalContainer.pedestal_std_hg) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(pedestalContainer.pedestal_std_lg) == ( + n_pixels, + N_SAMPLES, ) + j += 1 + # Check combined results + pedestalContainers = next( + NectarCAMPedestalContainers.from_hdf5(outfile) + ) + group_name = "data_combined" + pedestalContainer = pedestalContainers.containers[group_name] + assert pedestalContainer.nsamples == N_SAMPLES + assert np.all(pedestalContainer.nevents == max_events[i]) + assert np.shape(pedestalContainer.pixels_id) == (n_pixels,) + assert pedestalContainer.ucts_timestamp_min == np.uint64( + expected_ucts_timestamp_min[i] + ) + assert pedestalContainer.ucts_timestamp_max == np.uint64( + expected_ucts_timestamp_max[i] + ) + assert np.shape(pedestalContainer.pedestal_mean_hg) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(pedestalContainer.pedestal_mean_lg) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(pedestalContainer.pedestal_std_hg) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(pedestalContainer.pedestal_std_lg) == ( + n_pixels, + N_SAMPLES, + ) + assert np.allclose(pedestalContainer.pedestal_mean_hg, 245.0, atol=20.0) + assert np.allclose(pedestalContainer.pedestal_mean_lg, 245.0, atol=20.0) + assert np.allclose(pedestalContainer.pedestal_std_hg, 10, atol=10) + assert np.allclose( + pedestalContainer.pedestal_std_lg, 2.5, atol=2.0 if i == 0 else 2.3 + ) def test_timesel(self): """ @@ -142,7 +159,7 @@ def test_timesel(self): max_events = [n_slices[0] * events_per_slice, 13] tmin = [1674462932637860000, 1715007113924900000] tmax = [1674462932695700000, 1715007123524921000] - for i, run in enumerate(runs["Run number"]): + for i, _ in enumerate(runs["Run number"]): run_number = runs["Run number"][i] run_file = runs["Run file"][i] n_pixels = runs["N pixels"][i] @@ -197,7 +214,7 @@ def test_WaveformsStdFilter(self): n_slices = [3, 2] events_per_slice = 10 max_events = [n_slices[0] * events_per_slice, 13] - for i, run in enumerate(runs["Run number"]): + for i, _ in enumerate(runs["Run number"]): run_number = runs["Run number"][i] run_file = runs["Run file"][i] n_pixels = runs["N pixels"][i] @@ -251,7 +268,7 @@ def test_ChargeDistributionFilter(self): n_slices = [2, 1] events_per_slice = 10 max_events = [n_slices[0] * events_per_slice - 1, 12] - for i, run in enumerate(runs["Run number"]): + for i, _ in enumerate(runs["Run number"]): run_number = runs["Run number"][i] run_file = runs["Run file"][i] n_pixels = runs["N pixels"][i] @@ -358,7 +375,7 @@ def test_pixel_mask(self): pixel_mask_mean_max=1100.0, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -390,7 +407,7 @@ def test_pixel_mask(self): pixel_mask_std_sample_min=100.0, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() From c9f331c0c9ff0fe788ea89f6059a327cd6488f2a Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 9 Jan 2025 09:39:08 +0100 Subject: [PATCH 32/36] remove LightNectarCamEventSource --- src/nectarchain/data/container/eventSource.py | 255 ------------------ 1 file changed, 255 deletions(-) delete mode 100644 src/nectarchain/data/container/eventSource.py diff --git a/src/nectarchain/data/container/eventSource.py b/src/nectarchain/data/container/eventSource.py deleted file mode 100644 index ab8ce688..00000000 --- a/src/nectarchain/data/container/eventSource.py +++ /dev/null @@ -1,255 +0,0 @@ -import logging -import struct - -import numpy as np -from ctapipe.containers import EventType -from ctapipe_io_nectarcam import ( - NectarCAMDataContainer, - NectarCAMEventSource, - TriggerBits, - time_from_unix_tai_ns, -) -from ctapipe_io_nectarcam.anyarray_dtypes import ( - CDTS_AFTER_37201_DTYPE, - CDTS_BEFORE_37201_DTYPE, - TIB_DTYPE, -) -from ctapipe_io_nectarcam.constants import N_PIXELS -from ctapipe_io_nectarcam.containers import NectarCAMEventContainer - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - - -__all__ = ["LightNectarCAMEventSource"] - - -def fill_nectarcam_event_container_from_zfile(self, array_event, event): - """Fill the NectarCAM event container from the zfile event data. - - Parameters: - - array_event: The NectarCAMDataContainer object to fill with event data. - - event: The event data from the zfile. - - Returns: - - None - - This function fills the NectarCAM event container in the NectarCAMDataContainer - object with the event data from the zfile. It unpacks the necessary data from the - event and assigns it to the corresponding fields in the event container. - - The function performs the following steps: - 1. Assigns the tel_id to the local variable. - 2. Creates a new NectarCAMEventContainer object and assigns it to the - event_container field of the NectarCAMDataContainer object. - 3. Assigns the extdevices_presence field of the event to the extdevices_presence - field of the event_container. - 4. Assigns the counters field of the event to the counters field of the - event_container. - 5. Unpacks the TIB data from the event and assigns it to the corresponding fields in - the event_container. - 6. Unpacks the CDTS data from the event and assigns it to the corresponding fields - in the event_container. - 7. Calls the unpack_feb_data function to unpack the FEB counters and trigger pattern - from the event and assign them to the corresponding fields in the event_container. - """ - - tel_id = self.tel_id - event_container = NectarCAMEventContainer() - array_event.nectarcam.tel[tel_id].evt = event_container - event_container.extdevices_presence = event.nectarcam.extdevices_presence - event_container.counters = event.nectarcam.counters - # unpack TIB data - unpacked_tib = event.nectarcam.tib_data.view(TIB_DTYPE)[0] - event_container.tib_masked_trigger = unpacked_tib[4] - # unpack CDTS data - is_old_cdts = len(event.nectarcam.cdts_data) < 36 - if is_old_cdts: - unpacked_cdts = event.nectarcam.cdts_data.view(CDTS_BEFORE_37201_DTYPE)[0] - event_container.ucts_event_counter = unpacked_cdts[0] - event_container.ucts_timestamp = unpacked_cdts[3] - event_container.ucts_trigger_type = unpacked_cdts[5] - else: - unpacked_cdts = event.nectarcam.cdts_data.view(CDTS_AFTER_37201_DTYPE)[0] - event_container.ucts_timestamp = unpacked_cdts[0] - event_container.ucts_event_counter = unpacked_cdts[2] - event_container.ucts_busy_counter = unpacked_cdts[3] - event_container.ucts_trigger_type = unpacked_cdts[6] - # Unpack FEB counters and trigger pattern - self.unpack_feb_data(event_container, event) - - -def unpack_feb_data(self, event_container, event): - """Unpack FEB counters and trigger pattern.""" - # Deduce data format version - bytes_per_module = ( - len(event.nectarcam.counters) // self.nectarcam_service.num_modules - ) - # Remain compatible with data before addition of trigger pattern - module_fmt = "IHHIBBBBBBBB" if bytes_per_module > 16 else "IHHIBBBB" - n_fields = len(module_fmt) - rec_fmt = "=" + module_fmt * self.nectarcam_service.num_modules - # Unpack - unpacked_feb = struct.unpack(rec_fmt, event.nectarcam.counters) - # Initialize field containers - if bytes_per_module > 16: - n_patterns = 4 - event_container.trigger_pattern = np.zeros( - shape=(n_patterns, N_PIXELS), dtype=bool - ) - - for module_idx, module_id in enumerate(self.nectarcam_service.module_ids): - offset = module_id * 7 - if bytes_per_module > 16: - field_id = 8 - # Decode trigger pattern - for pattern_id in range(n_patterns): - value = unpacked_feb[n_fields * module_idx + field_id + pattern_id] - module_pattern = [ - int(digit) for digit in reversed(bin(value)[2:].zfill(7)) - ] - event_container.trigger_pattern[ - pattern_id, offset : offset + 7 - ] = module_pattern - - -def fill_trigger_info(self, array_event): - """Fill the trigger information for a given event. - - Parameters: - array_event (NectarCAMEventContainer): The NectarCAMEventContainer object to - fill with trigger information. - - Returns: - None - - Raises: - None - """ - - tel_id = self.tel_id - nectarcam = array_event.nectarcam.tel[tel_id] - tib_available = nectarcam.evt.extdevices_presence & 1 - ucts_available = nectarcam.evt.extdevices_presence & 2 - # fill trigger time using UCTS timestamp - trigger = array_event.trigger - trigger_time = nectarcam.evt.ucts_timestamp - trigger_time = time_from_unix_tai_ns(trigger_time) - trigger.time = trigger_time - trigger.tels_with_trigger = [tel_id] - trigger.tel[tel_id].time = trigger.time - # decide which source to use, if both are available, - # the option decides, if not, fallback to the avilable source - # if no source available, warn and do not fill trigger info - if tib_available and ucts_available: - if self.default_trigger_type == "ucts": - trigger_bits = nectarcam.evt.ucts_trigger_type - else: - trigger_bits = nectarcam.evt.tib_masked_trigger - elif tib_available: - trigger_bits = nectarcam.evt.tib_masked_trigger - elif ucts_available: - trigger_bits = nectarcam.evt.ucts_trigger_type - else: - self.log.warning("No trigger info available.") - trigger.event_type = EventType.UNKNOWN - return - if ( - ucts_available - and nectarcam.evt.ucts_trigger_type == 42 # TODO check if it's correct - and self.default_trigger_type == "ucts" - ): - self.log.warning( - "Event with UCTS trigger_type 42 found." - " Probably means unreliable or shifted UCTS data." - ' Consider switching to TIB using `default_trigger_type="tib"`' - ) - # first bit mono trigger, second stereo. - # If *only* those two are set, we assume it's a physics event - # for all other we only check if the flag is present - if (trigger_bits & TriggerBits.PHYSICS) and not (trigger_bits & TriggerBits.OTHER): - trigger.event_type = EventType.SUBARRAY - elif trigger_bits & TriggerBits.CALIBRATION: - trigger.event_type = EventType.FLATFIELD - elif trigger_bits & TriggerBits.PEDESTAL: - trigger.event_type = EventType.SKY_PEDESTAL - elif trigger_bits & TriggerBits.SINGLE_PE: - trigger.event_type = EventType.SINGLE_PE - else: - self.log.warning( - f"Event {array_event.index.event_id} has unknown event type, trigger: " - f"{trigger_bits:08b}" - ) - trigger.event_type = EventType.UNKNOWN - - -class LightNectarCAMEventSource(NectarCAMEventSource): - """LightNectarCAMEventSource is a subclass of NectarCAMEventSource that - provides a generator for iterating over NectarCAM events. - - This implementation of the NectarCAMEventSource is much lighter than the one within - ctapipe_io_nectarcam, only the fields interesting for nectarchain are kept. - - Parameters - ---------- - input_url : str - The input URL of the data source. - max_events : int - The maximum number of events to process. - tel_id : int - The telescope ID. - nectarcam_service : NectarCAMService - The service container for NectarCAM. - trigger_information : bool - Flag indicating whether to fill trigger information in the event container. - obs_ids : list - The list of observation IDs. - multi_file : MultiFileReader - The multi-file reader for reading the data source. - r0_r1_calibrator : R0R1Calibrator - The calibrator for R0 to R1 conversion. - calibrate_flatfields_and_pedestals : bool - Flag indicating whether to calibrate flatfield and pedestal events. - """ - - def _generator(self): - """The generator function that yields NectarCAMDataContainer objects - representing each event. - - Yields - ------ - NectarCAMDataContainer : - The NectarCAMDataContainer object representing each event. - - Raises - ------ - None - """ - # container for NectarCAM data - array_event = NectarCAMDataContainer() - array_event.meta["input_url"] = self.input_url - array_event.meta["max_events"] = self.max_events - array_event.meta["origin"] = "NectarCAM" - - # also add service container to the event section - array_event.nectarcam.tel[self.tel_id].svc = self.nectarcam_service - - # initialize general monitoring container - self.initialize_mon_container(array_event) - - # loop on events - for count, event in enumerate(self.multi_file): - array_event.count = count - array_event.index.event_id = event.event_id - array_event.index.obs_id = self.obs_ids[0] - - # fill R0/R1 data - self.fill_r0r1_container(array_event, event) - # fill specific NectarCAM event data - self.fill_nectarcam_event_container_from_zfile(array_event, event) - - if self.trigger_information: - self.fill_trigger_info(array_event) - - yield array_event From 617d8f22f71dae4ba046fb72c4b46d52d2bf9d7e Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 9 Jan 2025 11:40:55 +0100 Subject: [PATCH 33/36] bugfix in toml file introduced in previous commits --- pyproject.toml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 36e93a1a..93ae88c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,10 +45,6 @@ test = [ dev = [ "setuptools_scm", ] - -[tool.pytest.ini_options] -addopts = "--ignore=src/nectarchain/user_scripts" - docs = [ "sphinx", "sphinx-autodoc-typehints", @@ -58,12 +54,17 @@ docs = [ "numpydoc", "tomli; python_version < '3.11'" ] - # we can use self-references to simplify all all = [ "nectarchain[test,docs,dev]", ] + +[tool.pytest.ini_options] +addopts = "--ignore=src/nectarchain/user_scripts" + + + [tool.setuptools.packages.find] where = ["src"] exclude = ["nectarchain._dev_version"] From a9df416eac1b56b0880df0fa482ddfd0ad7729a7 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 9 Jan 2025 11:46:02 +0100 Subject: [PATCH 34/36] bugfix : __init__ file in tests repertory --- src/nectarchain/makers/tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/nectarchain/makers/tests/__init__.py diff --git a/src/nectarchain/makers/tests/__init__.py b/src/nectarchain/makers/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 From 36a773786937e3d25d9aab820bfad3a6babdc912 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 9 Jan 2025 11:56:42 +0100 Subject: [PATCH 35/36] pytest don't like to have 2 test files with same names --- .../data/container/tests/{test_core.py => test_core_container.py} | 0 .../makers/tests/{test_core.py => test_core_makers.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/nectarchain/data/container/tests/{test_core.py => test_core_container.py} (100%) rename src/nectarchain/makers/tests/{test_core.py => test_core_makers.py} (100%) diff --git a/src/nectarchain/data/container/tests/test_core.py b/src/nectarchain/data/container/tests/test_core_container.py similarity index 100% rename from src/nectarchain/data/container/tests/test_core.py rename to src/nectarchain/data/container/tests/test_core_container.py diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core_makers.py similarity index 100% rename from src/nectarchain/makers/tests/test_core.py rename to src/nectarchain/makers/tests/test_core_makers.py From 6dd1e8d4eea163fe0a1a20f00d81fbc1242fdc8a Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 9 Jan 2025 18:20:13 +0100 Subject: [PATCH 36/36] specific noqa --- src/nectarchain/data/container/core.py | 2 +- src/nectarchain/dqm/charge_integration.py | 14 +++++++------- src/nectarchain/makers/charges_makers.py | 18 ++++++++++-------- .../component/flatfield_spe_component.py | 10 +++++----- .../component/photostatistic_component.py | 2 +- .../ggrolleron/gain_PhotoStat_computation.py | 2 +- 6 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index 726f6e49..32384536 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -261,7 +261,7 @@ def _container_from_hdf5( """ if isinstance(path, str): path = Path(path) - module = importlib.import_module(f"{container_class.__module__}") # noqa + module = importlib.import_module(f"{container_class.__module__}") # noqa :F841 container = eval(f"module.{container_class.__name__}")() with HDF5TableReader(path) as reader: diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index a7e4a786..6018536f 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -1,13 +1,13 @@ import ctapipe.instrument.camera.readout import numpy as np from ctapipe.coordinates import EngineeringCameraFrame -from ctapipe.image.extractor import FixedWindowSum # noqa -from ctapipe.image.extractor import FullWaveformSum # noqa -from ctapipe.image.extractor import GlobalPeakWindowSum # noqa -from ctapipe.image.extractor import LocalPeakWindowSum # noqa -from ctapipe.image.extractor import NeighborPeakWindowSum # noqa -from ctapipe.image.extractor import SlidingWindowMaxSum # noqa -from ctapipe.image.extractor import TwoPassWindowSum # noqa +from ctapipe.image.extractor import FixedWindowSum # noqa: F401 +from ctapipe.image.extractor import FullWaveformSum # noqa: F401 +from ctapipe.image.extractor import GlobalPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import LocalPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import NeighborPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import SlidingWindowMaxSum # noqa: F401 +from ctapipe.image.extractor import TwoPassWindowSum # noqa: F401 from ctapipe.visualization import CameraDisplay from ctapipe_io_nectarcam import constants from matplotlib import pyplot as plt diff --git a/src/nectarchain/makers/charges_makers.py b/src/nectarchain/makers/charges_makers.py index fe5905f4..3998ea75 100644 --- a/src/nectarchain/makers/charges_makers.py +++ b/src/nectarchain/makers/charges_makers.py @@ -4,14 +4,16 @@ import numpy as np from ctapipe.core.traits import Bool, ComponentNameList -from ctapipe.image.extractor import BaselineSubtractedNeighborPeakWindowSum # noqa -from ctapipe.image.extractor import FixedWindowSum # noqa -from ctapipe.image.extractor import FullWaveformSum # noqa -from ctapipe.image.extractor import GlobalPeakWindowSum # noqa -from ctapipe.image.extractor import LocalPeakWindowSum # noqa -from ctapipe.image.extractor import NeighborPeakWindowSum # noqa -from ctapipe.image.extractor import SlidingWindowMaxSum # noqa -from ctapipe.image.extractor import TwoPassWindowSum # noqa +from ctapipe.image.extractor import FixedWindowSum # noqa: F401 +from ctapipe.image.extractor import FullWaveformSum # noqa: F401 +from ctapipe.image.extractor import GlobalPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import LocalPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import NeighborPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import SlidingWindowMaxSum # noqa: F401 +from ctapipe.image.extractor import TwoPassWindowSum # noqa: F401 +from ctapipe.image.extractor import ( # noqa: F401 + BaselineSubtractedNeighborPeakWindowSum, +) from ..data.container import WaveformsContainer, WaveformsContainers from ..data.management import DataManagement diff --git a/src/nectarchain/makers/component/flatfield_spe_component.py b/src/nectarchain/makers/component/flatfield_spe_component.py index f8a75088..8a09ee42 100644 --- a/src/nectarchain/makers/component/flatfield_spe_component.py +++ b/src/nectarchain/makers/component/flatfield_spe_component.py @@ -9,11 +9,11 @@ from ...utils import ComponentUtils from .charges_component import ChargesComponent from .gain_component import GainNectarCAMComponent -from .spe import SPECombinedalgorithm # noqa -from .spe import SPEHHValgorithm # noqa -from .spe import SPEHHVStdalgorithm # noqa -from .spe import SPEnominalalgorithm # noqa -from .spe import SPEnominalStdalgorithm # noqa +from .spe import SPECombinedalgorithm # noqa: F401 +from .spe import SPEHHValgorithm # noqa: F401 +from .spe import SPEHHVStdalgorithm # noqa: F401 +from .spe import SPEnominalalgorithm # noqa: F401 +from .spe import SPEnominalStdalgorithm # noqa: F401 logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) diff --git a/src/nectarchain/makers/component/photostatistic_component.py b/src/nectarchain/makers/component/photostatistic_component.py index 8c3982a1..5674fcca 100644 --- a/src/nectarchain/makers/component/photostatistic_component.py +++ b/src/nectarchain/makers/component/photostatistic_component.py @@ -10,7 +10,7 @@ from ...utils import ComponentUtils from .charges_component import ChargesComponent from .gain_component import GainNectarCAMComponent -from .photostatistic_algorithm import PhotoStatisticAlgorithm # noqa +from .photostatistic_algorithm import PhotoStatisticAlgorithm # noqa: F401 logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py index 20b72def..4261f903 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py @@ -138,7 +138,7 @@ def main( path = DataManagement.find_SPE_nominal( run_number=args.HHV_run_number, method="GlobalPeakWindowSum", - str_extractor_kwargs=f"window_width_{8}_window_shift_4", + str_extractor_kwargs=f"window_width_8_window_shift_4", ) if len(path) == 1: log.info(