From 25634d4950eba2e1ec1afb61e3ceda45cddd15ab Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Sat, 9 Nov 2024 18:05:21 -0300 Subject: [PATCH 01/19] Add a new flat field container --- src/nectarchain/data/container/flatfieldContainer.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/nectarchain/data/container/flatfieldContainer.py diff --git a/src/nectarchain/data/container/flatfieldContainer.py b/src/nectarchain/data/container/flatfieldContainer.py new file mode 100644 index 00000000..e4ceaa5e --- /dev/null +++ b/src/nectarchain/data/container/flatfieldContainer.py @@ -0,0 +1,5 @@ +import logging + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers From 727d2354f36a226488f173b56c616bd9e8258558 Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Mon, 11 Nov 2024 09:13:46 -0300 Subject: [PATCH 02/19] Add all necessary fields in FlatFieldContainer --- .../data/container/flatfieldContainer.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/src/nectarchain/data/container/flatfieldContainer.py b/src/nectarchain/data/container/flatfieldContainer.py index e4ceaa5e..2ebb8e72 100644 --- a/src/nectarchain/data/container/flatfieldContainer.py +++ b/src/nectarchain/data/container/flatfieldContainer.py @@ -3,3 +3,83 @@ logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers + +import numpy as np +from ctapipe.containers import Field +from ctapipe.core.traits import ComponentNameList, Dict, Integer, List, Tuple + +from .core import NectarCAMContainer + +__all__ = ["FlatFieldContainer"] + + +class FlatFieldContainer(NectarCAMContainer): + """ + Container that holds flat field coefficients and other useful information + + Fields: + run_number (np.uint16): Number of the run + npixels (np.uint16): Number of pixels + pixels_id (np.ndarray): Array of pixel's ID + ucts_timestamp (np.ndarray) : Array of time stamps of each event (UTC) + event_type (np.ndarray): Array of trigger event types (should be all flat field events) + event_id (np.ndarray): Array of the IDs of each event + amp_int_per_pix_per_event (np.ndarray): Array of integrated amplitude of each pulse + t_peak_per_pix_per_event (np.ndarray): Array of samples containing the pulse maximum + FF_coef (np.ndarray): Array of flat field coefficients + bad_pixels (List): List of pixel identified as outliers + """ + + run_number = Field( + type=np.uint16, + description="run number associated to the waveforms", + ) + + npixels = Field( + type=np.uint16, + description="number of effective pixels", + ) + + pixels_id = Field(type=np.ndarray, dtype=np.uint16, ndim=1, description="pixel ids") + + ucts_timestamp = Field( + type=np.ndarray, dtype=np.uint64, ndim=1, description="events ucts timestamp" + ) + + event_type = Field( + type=np.ndarray, dtype=np.uint8, ndim=1, description="trigger event type" + ) + + event_id = Field(type=np.ndarray, dtype=np.uint32, ndim=1, description="event ids") + + amp_int_per_pix_per_event = Field( + type=np.ndarray, + dtype=np.float32, + ndim=3, + description="The amplitude integrated over the window width, per pixel and per event", + ) + + t_peak_per_pix_per_event = Field( + type=np.ndarray, + dtype=np.uint8, + ndim=3, + description="Sample containing the pulse maximum, per pixel and per event", + ) + + FF_coef = Field( + type=np.ndarray, + dtype=np.float32, + ndim=3, + description="The flat field coefficient, per event", + ) + + # masked_wfs = Field( + # type=np.ndarray, dtype=np.uint64, ndim=4, description="Masked array for amplitude integration" + # ) + + bad_pixels = Field( + type=List, + dtype=np.uint16, + ndim=1, + description="Pixels considered as bad in at least one gain channels", + ) From ba476de6ea6a30cc736f55e524e0f401dd1b1078 Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Mon, 11 Nov 2024 09:31:16 -0300 Subject: [PATCH 03/19] Add new component for computation of flat field coefficient --- .../makers/component/preFlatFieldComponent.py | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 src/nectarchain/makers/component/preFlatFieldComponent.py diff --git a/src/nectarchain/makers/component/preFlatFieldComponent.py b/src/nectarchain/makers/component/preFlatFieldComponent.py new file mode 100644 index 00000000..20a66249 --- /dev/null +++ b/src/nectarchain/makers/component/preFlatFieldComponent.py @@ -0,0 +1,152 @@ +import logging + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + +import os +import pathlib + +import matplotlib.pyplot as plt +import numpy as np +from ctapipe.containers import EventType, Field +from ctapipe.coordinates import EngineeringCameraFrame +from ctapipe.core import Component +from ctapipe.core.traits import ComponentNameList, Dict, Float, Integer, List, Tuple +from ctapipe.instrument import CameraGeometry +from ctapipe.io import HDF5TableReader +from ctapipe.visualization import CameraDisplay +from ctapipe_io_nectarcam import constants +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer + +from nectarchain.data.container import ( + ArrayDataContainer, + NectarCAMContainer, + TriggerMapContainer, +) +from nectarchain.makers import EventsLoopNectarCAMCalibrationTool +from nectarchain.makers.component import ArrayDataComponent, NectarCAMComponent +from nectarchain.utils import ComponentUtils + +__all__ = ["preFlatFieldComponent"] + + +class preFlatFieldComponent(NectarCAMComponent): + window_shift = Integer( + default_value=5, + help="the time in ns before the peak to integrate charge", + ).tag(config=True) + + window_width = Integer( + default_value=12, + help="the duration of the extraction window in ns", + ).tag(config=True) + ## --< final window is 14 samples ! >-- + + g = Float( + default_value=58.0, + help="defaut gain value", + ).tag(config=True) + + hi_lo_ratio = Float( + default_value=13.0, + help="defaut gain value", + ).tag(config=True) + + ## Simple function to substract the pedestal using the 20 first samples of each trace + def substract_pedestal(wfs, window=20): + ped_mean = np.mean(wfs[0][:, :, 0:window], axis=2) + wfs_pedsub = wfs - np.expand_dims(ped_mean, axis=-1) + return wfs_pedsub + + ## Function to make an array that will be used as a mask on the waveform for the calculation of the integrated amplitude of the signal. + def make_masked_array(t_peak, window_shift, window_width): + masked_wfs = [ + np.array([np.zeros(constants.N_SAMPLES)] * constants.N_PIXELS) + ] * constants.N_GAINS + masked_wfs = np.array(masked_wfs) + + t_signal_start = t_peak - window_shift + t_signal_stop = t_peak + window_width - window_shift + + for g in range(0, constants.N_GAINS): + for i in range(0, constants.N_PIXELS): + masked_wfs[g][i][ + t_signal_start[0, g, i] : t_signal_stop[0, g, i] + ] = True + # --< I didn't find a better way to do than using this masked array >-- + + return masked_wfs + + def __init__(self, subarray, config=None, parent=None, *args, **kwargs): + super().__init__( + subarray=subarray, config=config, parent=parent, *args, **kwargs + ) + + self.__ucts_timestamp = [] + self.__event_type = [] + self.__event_id = [] + self.__amp_int_per_pix_per_event = [] + self.__FF_coef = [] + + def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): + wfs = [] + wfs_pedsub = [] + + if event.trigger.event_type.value == EventType.FLATFIELD.value: + # print("event :", (self.__event_id, self.__event_type)) + self.__event_id.append(np.uint32(event.index.event_id)) + self.__event_type.append(event.trigger.event_type.value) + self.__ucts_timestamp.append(event.nectarcam.tel[0].evt.ucts_timestamp) + + wfs.append(event.r0.tel[0].waveform) # not saved + + # substract pedestal using the mean of the 20 first samples + wfs_pedsub = substract_pedestal(wfs, 20) + + # get the masked array for integration window + t_peak = np.argmax(wfs_pedsub, axis=3) + masked_wfs = make_masked_array(t_peak, self.window_shift, self.window_width) + + # get integrated amplitude and mean amplitude over all pixels per event + amp_int_per_pix_per_event = np.sum( + wfs_pedsub[0], axis=2, where=masked_wfs.astype("bool") + ) + self.__amp_int_per_pix_per_event.append(amp_int_per_pix_per_event) + mean_amp_cam_per_event = np.mean(amp_int_per_pix_per_event, axis=-1) + + # get efficiency and flat field coefficient + gain = [self.g, self.g / self.hi_lo_ratio] + + eff = np.divide( + (amp_int_per_pix_per_event[:] / (np.expand_dims(gain[:], axis=-1))), + np.expand_dims((mean_amp_cam_per_event[:] / gain[:]), axis=-1), + ) # efficacité relative + FF_coef = np.ma.array(1.0 / eff, mask=eff == 0) + self.__FF_coef.append(FF_coef) + + def finish(self): + output = FlatFieldContainer( + run_number=FlatFieldContainer.fields["run_number"].type( + self._run_number + ), + npixels=FlatFieldContainer.fields["npixels"].type(self._npixels), + pixels_id=FlatFieldContainer.fields["pixels_id"].dtype.type( + self._pixels_id + ), + ucts_timestamp=FlatFieldContainer.fields["ucts_timestamp"].dtype.type( + self.__ucts_timestamp + ), + event_type=FlatFieldContainer.fields["event_type"].dtype.type( + self.__event_type + ), + event_id=FlatFieldContainer.fields["event_id"].dtype.type( + self.__event_id + ), + amp_int_per_pix_per_event=FlatFieldContainer.fields[ + "amp_int_per_pix_per_event" + ].dtype.type(self.__amp_int_per_pix_per_event), + FF_coef=FlatFieldContainer.fields["FF_coef"].dtype.type(self.__FF_coef), + ) + return output + From a7f6b5233ee8df6aceb87f57d70c51ae95256a13 Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Mon, 11 Nov 2024 09:34:40 -0300 Subject: [PATCH 04/19] Update flat field tools --- .../makers/calibration/flatfield_makers.py | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/nectarchain/makers/calibration/flatfield_makers.py b/src/nectarchain/makers/calibration/flatfield_makers.py index 046ce2de..36c3489d 100644 --- a/src/nectarchain/makers/calibration/flatfield_makers.py +++ b/src/nectarchain/makers/calibration/flatfield_makers.py @@ -10,9 +10,29 @@ __all__ = ["FlatfieldNectarCAMCalibrationTool"] -class FlatfieldNectarCAMCalibrationTool(NectarCAMCalibrationTool): - def start(self): - raise NotImplementedError( - "The computation of the flatfield calibration is not yet implemented, \ - feel free to contribute !:)" +#class FlatfieldNectarCAMCalibrationTool(NectarCAMCalibrationTool): +# def start(self): +# raise NotImplementedError( +# "The computation of the flatfield calibration is not yet implemented, feel free to contribute !:)" +# ) + +from nectarchain.makers import EventsLoopNectarCAMCalibrationTool + + +class FlatfieldNectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): + name = "NectarCAM" + + componentsList = ComponentNameList( + NectarCAMComponent, + default_value=["preFlatFieldComponent"], + help="List of Component names to be apply, the order will be respected", + ).tag(config=True) + + def _init_output_path(self): + if self.max_events is None: + filename = f"{self.name}_run{self.run_number}.h5" + else: + filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}.h5" + self.output_path = pathlib.Path( + f"{os.environ.get('NECTARCAMDATA','/tmp')}/tutorials/{filename}" ) From a51d3dde6eec488b907d86b85548d2fb03431ba0 Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Mon, 11 Nov 2024 10:15:04 -0300 Subject: [PATCH 05/19] New script for flat field tests --- .../user_scripts/ajardinb/FlatField_test.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/nectarchain/user_scripts/ajardinb/FlatField_test.py diff --git a/src/nectarchain/user_scripts/ajardinb/FlatField_test.py b/src/nectarchain/user_scripts/ajardinb/FlatField_test.py new file mode 100644 index 00000000..a121b196 --- /dev/null +++ b/src/nectarchain/user_scripts/ajardinb/FlatField_test.py @@ -0,0 +1,37 @@ +import os +import pathlib + +import matplotlib.pyplot as plt + +# %% +import numpy as np +from ctapipe.containers import Field +from ctapipe.core import Component +from ctapipe.core.traits import ComponentNameList, Integer, List, Dict, Tuple +from ctapipe.io import HDF5TableReader +from ctapipe_io_nectarcam import constants +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer +from ctapipe.instrument import CameraGeometry +from ctapipe.visualization import CameraDisplay +from ctapipe.coordinates import EngineeringCameraFrame + +from nectarchain.data.container import ( + ArrayDataContainer, + NectarCAMContainer, + TriggerMapContainer, +) +from ctapipe.containers import EventType +from nectarchain.makers import EventsLoopNectarCAMCalibrationTool +from nectarchain.makers.component import ArrayDataComponent, NectarCAMComponent +from nectarchain.utils import ComponentUtils + +from ctapipe.io import EventSource, EventSeeker +from ctapipe_io_nectarcam import NectarCAMEventSource +from astropy import units as u +from astropy.time import Time + +from scipy import signal +from scipy.signal import find_peaks +from scipy.interpolate import Rbf, InterpolatedUnivariateSpline +from scipy.stats import chi2, norm +import scipy From 2f8a02383319511f7e16f07d652d0db868b983c5 Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Fri, 22 Nov 2024 17:07:49 -0300 Subject: [PATCH 06/19] Correct output path --- src/nectarchain/makers/calibration/flatfield_makers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nectarchain/makers/calibration/flatfield_makers.py b/src/nectarchain/makers/calibration/flatfield_makers.py index 36c3489d..c0f1ca1e 100644 --- a/src/nectarchain/makers/calibration/flatfield_makers.py +++ b/src/nectarchain/makers/calibration/flatfield_makers.py @@ -34,5 +34,5 @@ def _init_output_path(self): else: filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}.h5" self.output_path = pathlib.Path( - f"{os.environ.get('NECTARCAMDATA','/tmp')}/tutorials/{filename}" + f"{os.environ.get('NECTARCAMDATA','/tmp')}/FlatFieldTests/{filename}" ) From 43d697b405afd67023d933238831568be4e586c6 Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Mon, 13 Jan 2025 18:11:33 -0300 Subject: [PATCH 07/19] Add flatfieldContainer and preFlatFieldComponent import --- src/nectarchain/data/container/__init__.py | 1 + src/nectarchain/makers/component/__init__.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/nectarchain/data/container/__init__.py b/src/nectarchain/data/container/__init__.py index 574b5ed1..651cdc14 100644 --- a/src/nectarchain/data/container/__init__.py +++ b/src/nectarchain/data/container/__init__.py @@ -33,3 +33,4 @@ "NectarCAMPedestalContainers", "PedestalFlagBits", ] +from .flatfieldContainer import * diff --git a/src/nectarchain/makers/component/__init__.py b/src/nectarchain/makers/component/__init__.py index 62aae577..5b38dc03 100644 --- a/src/nectarchain/makers/component/__init__.py +++ b/src/nectarchain/makers/component/__init__.py @@ -13,6 +13,8 @@ from .photostatistic_component import PhotoStatisticNectarCAMComponent from .spe import SPECombinedalgorithm, SPEHHValgorithm, SPEHHVStdalgorithm from .waveforms_component import WaveformsComponent +from .preFlatFieldComponent impor + __all__ = [ "ArrayDataComponent", @@ -32,4 +34,5 @@ "PhotoStatisticNectarCAMComponent", "PhotoStatisticAlgorithm", "GainNectarCAMComponent", + "preFlatFieldComponent", ] From 4da4e0f108c488bc2844184e6c2b14e35a7b2ea4 Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Tue, 14 Jan 2025 17:24:44 -0300 Subject: [PATCH 08/19] Add some import --- .../makers/calibration/flatfield_makers.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/nectarchain/makers/calibration/flatfield_makers.py b/src/nectarchain/makers/calibration/flatfield_makers.py index c0f1ca1e..a0b40c73 100644 --- a/src/nectarchain/makers/calibration/flatfield_makers.py +++ b/src/nectarchain/makers/calibration/flatfield_makers.py @@ -1,4 +1,11 @@ import logging +import os +import pathlib + +from ctapipe.core.traits import ComponentNameList + +from nectarchain.makers import EventsLoopNectarCAMCalibrationTool +from nectarchain.makers.component import NectarCAMComponent from .core import NectarCAMCalibrationTool @@ -6,19 +13,11 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers +from .core import NectarCAMCalibrationTool __all__ = ["FlatfieldNectarCAMCalibrationTool"] -#class FlatfieldNectarCAMCalibrationTool(NectarCAMCalibrationTool): -# def start(self): -# raise NotImplementedError( -# "The computation of the flatfield calibration is not yet implemented, feel free to contribute !:)" -# ) - -from nectarchain.makers import EventsLoopNectarCAMCalibrationTool - - class FlatfieldNectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): name = "NectarCAM" @@ -36,3 +35,4 @@ def _init_output_path(self): self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/FlatFieldTests/{filename}" ) + From 005d4da203e66444d5fff76c1ebf45565bf827f9 Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Tue, 14 Jan 2025 17:32:57 -0300 Subject: [PATCH 09/19] Clean imports --- .../data/container/flatfieldContainer.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/nectarchain/data/container/flatfieldContainer.py b/src/nectarchain/data/container/flatfieldContainer.py index 2ebb8e72..ebe3a2e1 100644 --- a/src/nectarchain/data/container/flatfieldContainer.py +++ b/src/nectarchain/data/container/flatfieldContainer.py @@ -1,15 +1,15 @@ import logging -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import numpy as np from ctapipe.containers import Field -from ctapipe.core.traits import ComponentNameList, Dict, Integer, List, Tuple +from ctapipe.core.traits import List from .core import NectarCAMContainer +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + __all__ = ["FlatFieldContainer"] @@ -22,10 +22,13 @@ class FlatFieldContainer(NectarCAMContainer): npixels (np.uint16): Number of pixels pixels_id (np.ndarray): Array of pixel's ID ucts_timestamp (np.ndarray) : Array of time stamps of each event (UTC) - event_type (np.ndarray): Array of trigger event types (should be all flat field events) + event_type (np.ndarray): Array of trigger event types (should be all flat + field events) event_id (np.ndarray): Array of the IDs of each event - amp_int_per_pix_per_event (np.ndarray): Array of integrated amplitude of each pulse - t_peak_per_pix_per_event (np.ndarray): Array of samples containing the pulse maximum + amp_int_per_pix_per_event (np.ndarray): Array of integrated amplitude of each + pulse + t_peak_per_pix_per_event (np.ndarray): Array of samples containing the pulse + maximum FF_coef (np.ndarray): Array of flat field coefficients bad_pixels (List): List of pixel identified as outliers """ @@ -56,30 +59,33 @@ class FlatFieldContainer(NectarCAMContainer): type=np.ndarray, dtype=np.float32, ndim=3, - description="The amplitude integrated over the window width, per pixel and per event", + description="amplitude integrated over the window width, per pixel per event", ) t_peak_per_pix_per_event = Field( type=np.ndarray, dtype=np.uint8, ndim=3, - description="Sample containing the pulse maximum, per pixel and per event", + description="sample containing the pulse maximum, per pixel and per event", ) FF_coef = Field( type=np.ndarray, dtype=np.float32, ndim=3, - description="The flat field coefficient, per event", + description="the flat field coefficients, per event", ) # masked_wfs = Field( - # type=np.ndarray, dtype=np.uint64, ndim=4, description="Masked array for amplitude integration" + # type=np.ndarray, + # dtype=np.uint64, + # ndim=4, + # description="Masked array for amplitude integration", # ) bad_pixels = Field( type=List, dtype=np.uint16, ndim=1, - description="Pixels considered as bad in at least one gain channels", + description="pixels considered as bad in at least one gain channels", ) From 2a5951215b6fe2f68cf2d0979e0281e2a8f1d600 Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Tue, 14 Jan 2025 17:47:28 -0300 Subject: [PATCH 10/19] implement of comments and cleaning --- .../makers/component/preFlatFieldComponent.py | 161 +++++++++++------- 1 file changed, 96 insertions(+), 65 deletions(-) diff --git a/src/nectarchain/makers/component/preFlatFieldComponent.py b/src/nectarchain/makers/component/preFlatFieldComponent.py index 20a66249..eeacb32b 100644 --- a/src/nectarchain/makers/component/preFlatFieldComponent.py +++ b/src/nectarchain/makers/component/preFlatFieldComponent.py @@ -1,37 +1,41 @@ import logging -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - -import os -import pathlib - -import matplotlib.pyplot as plt import numpy as np -from ctapipe.containers import EventType, Field -from ctapipe.coordinates import EngineeringCameraFrame -from ctapipe.core import Component -from ctapipe.core.traits import ComponentNameList, Dict, Float, Integer, List, Tuple -from ctapipe.instrument import CameraGeometry -from ctapipe.io import HDF5TableReader -from ctapipe.visualization import CameraDisplay +from ctapipe.containers import EventType +from ctapipe.core.traits import Float, Integer from ctapipe_io_nectarcam import constants from ctapipe_io_nectarcam.containers import NectarCAMDataContainer -from nectarchain.data.container import ( - ArrayDataContainer, - NectarCAMContainer, - TriggerMapContainer, -) -from nectarchain.makers import EventsLoopNectarCAMCalibrationTool -from nectarchain.makers.component import ArrayDataComponent, NectarCAMComponent -from nectarchain.utils import ComponentUtils +from nectarchain.data.container import FlatFieldContainer +from nectarchain.makers.component import NectarCAMComponent + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers __all__ = ["preFlatFieldComponent"] class preFlatFieldComponent(NectarCAMComponent): + """ + Component that computes flat field coefficients from raw data. + + Parameters + ---------- + window_shift: int + time in ns before the peak to integrate charge (default value = 5) + + window_width: int + duration of the extraction window in ns (default value = 12) + + g: float + default gain value (default value = 58) + + hi_lo_ratio: float + default high gain to low gain ratio (default value = 13) + + """ + window_shift = Integer( default_value=5, help="the time in ns before the peak to integrate charge", @@ -41,43 +45,18 @@ class preFlatFieldComponent(NectarCAMComponent): default_value=12, help="the duration of the extraction window in ns", ).tag(config=True) - ## --< final window is 14 samples ! >-- + # --< final window is 14 samples ! >-- g = Float( default_value=58.0, - help="defaut gain value", + help="default gain value", ).tag(config=True) hi_lo_ratio = Float( default_value=13.0, - help="defaut gain value", + help="default high gain to low gain ratio", ).tag(config=True) - ## Simple function to substract the pedestal using the 20 first samples of each trace - def substract_pedestal(wfs, window=20): - ped_mean = np.mean(wfs[0][:, :, 0:window], axis=2) - wfs_pedsub = wfs - np.expand_dims(ped_mean, axis=-1) - return wfs_pedsub - - ## Function to make an array that will be used as a mask on the waveform for the calculation of the integrated amplitude of the signal. - def make_masked_array(t_peak, window_shift, window_width): - masked_wfs = [ - np.array([np.zeros(constants.N_SAMPLES)] * constants.N_PIXELS) - ] * constants.N_GAINS - masked_wfs = np.array(masked_wfs) - - t_signal_start = t_peak - window_shift - t_signal_stop = t_peak + window_width - window_shift - - for g in range(0, constants.N_GAINS): - for i in range(0, constants.N_PIXELS): - masked_wfs[g][i][ - t_signal_start[0, g, i] : t_signal_stop[0, g, i] - ] = True - # --< I didn't find a better way to do than using this masked array >-- - - return masked_wfs - def __init__(self, subarray, config=None, parent=None, *args, **kwargs): super().__init__( subarray=subarray, config=config, parent=parent, *args, **kwargs @@ -101,35 +80,90 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): wfs.append(event.r0.tel[0].waveform) # not saved - # substract pedestal using the mean of the 20 first samples - wfs_pedsub = substract_pedestal(wfs, 20) + # subtract pedestal using the mean of the 20 first samples + wfs_pedsub = self.subtract_pedestal(wfs, 20) # get the masked array for integration window t_peak = np.argmax(wfs_pedsub, axis=3) - masked_wfs = make_masked_array(t_peak, self.window_shift, self.window_width) + masked_wfs = self.make_masked_array( + t_peak, self.window_shift, self.window_width + ) + # --< I didn't find a better way to do than using this masked array >-- # get integrated amplitude and mean amplitude over all pixels per event amp_int_per_pix_per_event = np.sum( wfs_pedsub[0], axis=2, where=masked_wfs.astype("bool") ) self.__amp_int_per_pix_per_event.append(amp_int_per_pix_per_event) - mean_amp_cam_per_event = np.mean(amp_int_per_pix_per_event, axis=-1) + # mean_amp_cam_per_event = np.mean(amp_int_per_pix_per_event, axis=-1) # get efficiency and flat field coefficient gain = [self.g, self.g / self.hi_lo_ratio] + amp_int_per_pix_per_event_pe = amp_int_per_pix_per_event[:] / ( + np.expand_dims(gain[:], axis=-1) + ) + mean_amp_cam_per_event_pe = np.mean(amp_int_per_pix_per_event_pe, axis=-1) + eff = np.divide( - (amp_int_per_pix_per_event[:] / (np.expand_dims(gain[:], axis=-1))), - np.expand_dims((mean_amp_cam_per_event[:] / gain[:]), axis=-1), - ) # efficacité relative + amp_int_per_pix_per_event_pe, + np.expand_dims(mean_amp_cam_per_event_pe, axis=-1), + ) + FF_coef = np.ma.array(1.0 / eff, mask=eff == 0) self.__FF_coef.append(FF_coef) + @staticmethod + def subtract_pedestal(wfs, window=20): + """ + Subtract the pedestal defined as the average of the first samples of each trace + + Args: + wfs: raw wavefroms + window: number of samples n to calculate the pedestal (default value is 20) + + Returns: + wfs_pedsub: wavefroms subtracted from the pedestal + """ + + ped_mean = np.mean(wfs[0][:, :, 0:window], axis=2) + wfs_pedsub = wfs - np.expand_dims(ped_mean, axis=-1) + + return wfs_pedsub + + @staticmethod + def make_masked_array(t_peak, window_shift, window_width): + """ + Define an array that will be used as a mask on the waveforms for the calculation + of the integrated amplitude of the signal + + Args: + t_peak: time corresponding the the highest peak of the trace + window_shift: time in ns before the peak to integrate charge + window_width: duration of the extraction window in ns + + Returns: + masked_wfs: a mask array + """ + + masked_wfs = np.zeros( + shape=(constants.N_GAINS, constants.N_PIXELS, constants.N_SAMPLES), + dtype=bool, + ) + t_signal_start = t_peak - window_shift + t_signal_stop = t_peak + window_width - window_shift + + for g in range(0, constants.N_GAINS): + for i in range(0, constants.N_PIXELS): + masked_wfs[g][i][ + t_signal_start[0, g, i] : t_signal_stop[0, g, i] + ] = True + + return masked_wfs + def finish(self): output = FlatFieldContainer( - run_number=FlatFieldContainer.fields["run_number"].type( - self._run_number - ), + run_number=FlatFieldContainer.fields["run_number"].type(self._run_number), npixels=FlatFieldContainer.fields["npixels"].type(self._npixels), pixels_id=FlatFieldContainer.fields["pixels_id"].dtype.type( self._pixels_id @@ -140,13 +174,10 @@ def finish(self): event_type=FlatFieldContainer.fields["event_type"].dtype.type( self.__event_type ), - event_id=FlatFieldContainer.fields["event_id"].dtype.type( - self.__event_id - ), + event_id=FlatFieldContainer.fields["event_id"].dtype.type(self.__event_id), amp_int_per_pix_per_event=FlatFieldContainer.fields[ "amp_int_per_pix_per_event" ].dtype.type(self.__amp_int_per_pix_per_event), FF_coef=FlatFieldContainer.fields["FF_coef"].dtype.type(self.__FF_coef), ) return output - From 4e35249f78b620fe18dab65694821c0297a668d5 Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Tue, 14 Jan 2025 17:50:58 -0300 Subject: [PATCH 11/19] example script to run the flat field tool --- .../user_scripts/ajardinb/FlatField_test.py | 49 +++++++------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/src/nectarchain/user_scripts/ajardinb/FlatField_test.py b/src/nectarchain/user_scripts/ajardinb/FlatField_test.py index a121b196..7bc2e277 100644 --- a/src/nectarchain/user_scripts/ajardinb/FlatField_test.py +++ b/src/nectarchain/user_scripts/ajardinb/FlatField_test.py @@ -1,37 +1,26 @@ import os -import pathlib -import matplotlib.pyplot as plt +from nectarchain.makers.calibration import FlatfieldNectarCAMCalibrationTool -# %% -import numpy as np -from ctapipe.containers import Field -from ctapipe.core import Component -from ctapipe.core.traits import ComponentNameList, Integer, List, Dict, Tuple -from ctapipe.io import HDF5TableReader -from ctapipe_io_nectarcam import constants -from ctapipe_io_nectarcam.containers import NectarCAMDataContainer -from ctapipe.instrument import CameraGeometry -from ctapipe.visualization import CameraDisplay -from ctapipe.coordinates import EngineeringCameraFrame +# Define the global environment variable NECTARCAMDATA (folder where are the runs) +os.environ["NECTARCAMDATA"] = "./20231222" -from nectarchain.data.container import ( - ArrayDataContainer, - NectarCAMContainer, - TriggerMapContainer, +run_number = 4940 +max_events = 10000 +window_width = 14 + +# Call the tool +tool = FlatfieldNectarCAMCalibrationTool( + progress_bar=True, + run_number=run_number, + max_events=max_events, + log_level=20, + window_width=window_width, + overwrite=True, ) -from ctapipe.containers import EventType -from nectarchain.makers import EventsLoopNectarCAMCalibrationTool -from nectarchain.makers.component import ArrayDataComponent, NectarCAMComponent -from nectarchain.utils import ComponentUtils -from ctapipe.io import EventSource, EventSeeker -from ctapipe_io_nectarcam import NectarCAMEventSource -from astropy import units as u -from astropy.time import Time +tool.initialize() +tool.setup() -from scipy import signal -from scipy.signal import find_peaks -from scipy.interpolate import Rbf, InterpolatedUnivariateSpline -from scipy.stats import chi2, norm -import scipy +tool.start() +preFlatFieldOutput = tool.finish(return_output_component=True)[0] From 61e80315ea930168b3f35049e9a0fbfc935c442a Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Wed, 15 Jan 2025 09:43:34 -0300 Subject: [PATCH 12/19] correct the name --- src/nectarchain/makers/calibration/flatfield_makers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nectarchain/makers/calibration/flatfield_makers.py b/src/nectarchain/makers/calibration/flatfield_makers.py index a0b40c73..0275b8d0 100644 --- a/src/nectarchain/makers/calibration/flatfield_makers.py +++ b/src/nectarchain/makers/calibration/flatfield_makers.py @@ -19,7 +19,7 @@ class FlatfieldNectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): - name = "NectarCAM" + name = "FlatfieldNectarCAMCalibrationTool" componentsList = ComponentNameList( NectarCAMComponent, From cf1a1df22371e7e2807f22f0660ab653140e3468 Mon Sep 17 00:00:00 2001 From: jlenain Date: Thu, 16 Jan 2025 15:10:21 +0100 Subject: [PATCH 13/19] Fix conflicts following merge of PR #163 --- src/nectarchain/data/container/__init__.py | 3 ++- .../{flatfieldContainer.py => flatfield_container.py} | 0 src/nectarchain/makers/calibration/flatfield_makers.py | 7 +------ src/nectarchain/makers/component/__init__.py | 5 ++--- ...{preFlatFieldComponent.py => preflatfield_component.py} | 4 ++-- 5 files changed, 7 insertions(+), 12 deletions(-) rename src/nectarchain/data/container/{flatfieldContainer.py => flatfield_container.py} (100%) rename src/nectarchain/makers/component/{preFlatFieldComponent.py => preflatfield_component.py} (98%) diff --git a/src/nectarchain/data/container/__init__.py b/src/nectarchain/data/container/__init__.py index 651cdc14..ae110983 100644 --- a/src/nectarchain/data/container/__init__.py +++ b/src/nectarchain/data/container/__init__.py @@ -9,6 +9,7 @@ get_array_keys, merge_map_ArrayDataContainer, ) +from .flatfield_container import FlatFieldContainer from .gain_container import GainContainer, SPEfitContainer from .pedestal_container import ( NectarCAMPedestalContainer, @@ -32,5 +33,5 @@ "NectarCAMPedestalContainer", "NectarCAMPedestalContainers", "PedestalFlagBits", + "FlatFieldContainer", ] -from .flatfieldContainer import * diff --git a/src/nectarchain/data/container/flatfieldContainer.py b/src/nectarchain/data/container/flatfield_container.py similarity index 100% rename from src/nectarchain/data/container/flatfieldContainer.py rename to src/nectarchain/data/container/flatfield_container.py diff --git a/src/nectarchain/makers/calibration/flatfield_makers.py b/src/nectarchain/makers/calibration/flatfield_makers.py index 0275b8d0..c5c4b363 100644 --- a/src/nectarchain/makers/calibration/flatfield_makers.py +++ b/src/nectarchain/makers/calibration/flatfield_makers.py @@ -7,14 +7,10 @@ from nectarchain.makers import EventsLoopNectarCAMCalibrationTool from nectarchain.makers.component import NectarCAMComponent -from .core import NectarCAMCalibrationTool - logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from .core import NectarCAMCalibrationTool - __all__ = ["FlatfieldNectarCAMCalibrationTool"] @@ -23,7 +19,7 @@ class FlatfieldNectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): componentsList = ComponentNameList( NectarCAMComponent, - default_value=["preFlatFieldComponent"], + default_value=["PreFlatFieldComponent"], help="List of Component names to be apply, the order will be respected", ).tag(config=True) @@ -35,4 +31,3 @@ def _init_output_path(self): self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/FlatFieldTests/{filename}" ) - diff --git a/src/nectarchain/makers/component/__init__.py b/src/nectarchain/makers/component/__init__.py index 5b38dc03..3843da23 100644 --- a/src/nectarchain/makers/component/__init__.py +++ b/src/nectarchain/makers/component/__init__.py @@ -11,10 +11,9 @@ from .pedestal_component import PedestalEstimationComponent from .photostatistic_algorithm import PhotoStatisticAlgorithm from .photostatistic_component import PhotoStatisticNectarCAMComponent +from .preflatfield_component import PreFlatFieldComponent from .spe import SPECombinedalgorithm, SPEHHValgorithm, SPEHHVStdalgorithm from .waveforms_component import WaveformsComponent -from .preFlatFieldComponent impor - __all__ = [ "ArrayDataComponent", @@ -34,5 +33,5 @@ "PhotoStatisticNectarCAMComponent", "PhotoStatisticAlgorithm", "GainNectarCAMComponent", - "preFlatFieldComponent", + "PreFlatFieldComponent", ] diff --git a/src/nectarchain/makers/component/preFlatFieldComponent.py b/src/nectarchain/makers/component/preflatfield_component.py similarity index 98% rename from src/nectarchain/makers/component/preFlatFieldComponent.py rename to src/nectarchain/makers/component/preflatfield_component.py index eeacb32b..1b3cfde2 100644 --- a/src/nectarchain/makers/component/preFlatFieldComponent.py +++ b/src/nectarchain/makers/component/preflatfield_component.py @@ -13,10 +13,10 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -__all__ = ["preFlatFieldComponent"] +__all__ = ["PreFlatFieldComponent"] -class preFlatFieldComponent(NectarCAMComponent): +class PreFlatFieldComponent(NectarCAMComponent): """ Component that computes flat field coefficients from raw data. From 7cc41331b66faa47fd613a4efd72dbc157fd3dbd Mon Sep 17 00:00:00 2001 From: jlenain Date: Thu, 16 Jan 2025 15:59:31 +0100 Subject: [PATCH 14/19] Renamed script to avoid any conflict with pytest --- .../ajardinb/{FlatField_test.py => flatfield_example.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/nectarchain/user_scripts/ajardinb/{FlatField_test.py => flatfield_example.py} (92%) diff --git a/src/nectarchain/user_scripts/ajardinb/FlatField_test.py b/src/nectarchain/user_scripts/ajardinb/flatfield_example.py similarity index 92% rename from src/nectarchain/user_scripts/ajardinb/FlatField_test.py rename to src/nectarchain/user_scripts/ajardinb/flatfield_example.py index 7bc2e277..f0855c30 100644 --- a/src/nectarchain/user_scripts/ajardinb/FlatField_test.py +++ b/src/nectarchain/user_scripts/ajardinb/flatfield_example.py @@ -3,7 +3,7 @@ from nectarchain.makers.calibration import FlatfieldNectarCAMCalibrationTool # Define the global environment variable NECTARCAMDATA (folder where are the runs) -os.environ["NECTARCAMDATA"] = "./20231222" +# os.environ["NECTARCAMDATA"] = "./20231222" run_number = 4940 max_events = 10000 From 8effd79b6538485d5499cf3fc45cadfcbb0185b0 Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Fri, 17 Jan 2025 17:51:16 -0300 Subject: [PATCH 15/19] comment the field for peak time (not used) and change the bad pixel field from list to np ndarray --- .../data/container/flatfield_container.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/nectarchain/data/container/flatfield_container.py b/src/nectarchain/data/container/flatfield_container.py index ebe3a2e1..50e0a4ef 100644 --- a/src/nectarchain/data/container/flatfield_container.py +++ b/src/nectarchain/data/container/flatfield_container.py @@ -2,7 +2,6 @@ import numpy as np from ctapipe.containers import Field -from ctapipe.core.traits import List from .core import NectarCAMContainer @@ -62,12 +61,12 @@ class FlatFieldContainer(NectarCAMContainer): description="amplitude integrated over the window width, per pixel per event", ) - t_peak_per_pix_per_event = Field( - type=np.ndarray, - dtype=np.uint8, - ndim=3, - description="sample containing the pulse maximum, per pixel and per event", - ) + # t_peak_per_pix_per_event = Field( + # type=np.ndarray, + # dtype=np.float32, + # ndim=3, + # description="sample containing the pulse maximum, per pixel and per event", + # ) FF_coef = Field( type=np.ndarray, @@ -84,8 +83,8 @@ class FlatFieldContainer(NectarCAMContainer): # ) bad_pixels = Field( - type=List, + type=np.ndarray, dtype=np.uint16, - ndim=1, + ndim=2, description="pixels considered as bad in at least one gain channels", ) From 36868c378e9b89e51c95c55129a6110f37e5a91a Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Fri, 17 Jan 2025 17:53:55 -0300 Subject: [PATCH 16/19] implement changes to use the same components for the 2 passes --- .../component/preflatfield_component.py | 81 ++++++++++--------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/src/nectarchain/makers/component/preflatfield_component.py b/src/nectarchain/makers/component/preflatfield_component.py index 1b3cfde2..15189f77 100644 --- a/src/nectarchain/makers/component/preflatfield_component.py +++ b/src/nectarchain/makers/component/preflatfield_component.py @@ -2,7 +2,7 @@ import numpy as np from ctapipe.containers import EventType -from ctapipe.core.traits import Float, Integer +from ctapipe.core.traits import Integer, List from ctapipe_io_nectarcam import constants from ctapipe_io_nectarcam.containers import NectarCAMDataContainer @@ -28,11 +28,11 @@ class PreFlatFieldComponent(NectarCAMComponent): window_width: int duration of the extraction window in ns (default value = 12) - g: float - default gain value (default value = 58) + gain: list + array of gain value - hi_lo_ratio: float - default high gain to low gain ratio (default value = 13) + bad_pix: list + list of bad pixels (default value = []) """ @@ -45,16 +45,20 @@ class PreFlatFieldComponent(NectarCAMComponent): default_value=12, help="the duration of the extraction window in ns", ).tag(config=True) - # --< final window is 14 samples ! >-- - g = Float( - default_value=58.0, + gain = List( + default_value=None, help="default gain value", ).tag(config=True) - hi_lo_ratio = Float( - default_value=13.0, - help="default high gain to low gain ratio", + # hi_lo_ratio = Float( + # default_value=13.0, + # help="default high gain to low gain ratio", + # ).tag(config=True) + + bad_pix = List( + default_value=None, + help="list of bad pixels", ).tag(config=True) def __init__(self, subarray, config=None, parent=None, *args, **kwargs): @@ -67,42 +71,44 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): self.__event_id = [] self.__amp_int_per_pix_per_event = [] self.__FF_coef = [] + self.__bad_pixels = [] - def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): - wfs = [] - wfs_pedsub = [] + print("gain") + print(type(self.gain)) + print(self.gain) + + print("bad_pixels") + print(type(self.bad_pix)) + print(self.bad_pix) + def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): if event.trigger.event_type.value == EventType.FLATFIELD.value: # print("event :", (self.__event_id, self.__event_type)) self.__event_id.append(np.uint32(event.index.event_id)) self.__event_type.append(event.trigger.event_type.value) self.__ucts_timestamp.append(event.nectarcam.tel[0].evt.ucts_timestamp) - wfs.append(event.r0.tel[0].waveform) # not saved + wfs = event.r0.tel[0].waveform # subtract pedestal using the mean of the 20 first samples wfs_pedsub = self.subtract_pedestal(wfs, 20) # get the masked array for integration window - t_peak = np.argmax(wfs_pedsub, axis=3) + t_peak = np.argmax(wfs_pedsub, axis=-1) masked_wfs = self.make_masked_array( t_peak, self.window_shift, self.window_width ) - # --< I didn't find a better way to do than using this masked array >-- + + # mask bad pixels + self.__bad_pixels.append(self.bad_pix) + masked_wfs[:, self.bad_pix, :] = False # get integrated amplitude and mean amplitude over all pixels per event - amp_int_per_pix_per_event = np.sum( - wfs_pedsub[0], axis=2, where=masked_wfs.astype("bool") - ) + amp_int_per_pix_per_event = np.sum(wfs_pedsub, axis=-1, where=masked_wfs) self.__amp_int_per_pix_per_event.append(amp_int_per_pix_per_event) - # mean_amp_cam_per_event = np.mean(amp_int_per_pix_per_event, axis=-1) + # --< We could use ctapipe.image.extractor.LocalPeakWindowSum >-- - # get efficiency and flat field coefficient - gain = [self.g, self.g / self.hi_lo_ratio] - - amp_int_per_pix_per_event_pe = amp_int_per_pix_per_event[:] / ( - np.expand_dims(gain[:], axis=-1) - ) + amp_int_per_pix_per_event_pe = amp_int_per_pix_per_event[:] / self.gain[:] mean_amp_cam_per_event_pe = np.mean(amp_int_per_pix_per_event_pe, axis=-1) eff = np.divide( @@ -126,7 +132,7 @@ def subtract_pedestal(wfs, window=20): wfs_pedsub: wavefroms subtracted from the pedestal """ - ped_mean = np.mean(wfs[0][:, :, 0:window], axis=2) + ped_mean = np.mean(wfs[:, :, 0:window], axis=2) wfs_pedsub = wfs - np.expand_dims(ped_mean, axis=-1) return wfs_pedsub @@ -138,9 +144,9 @@ def make_masked_array(t_peak, window_shift, window_width): of the integrated amplitude of the signal Args: - t_peak: time corresponding the the highest peak of the trace - window_shift: time in ns before the peak to integrate charge - window_width: duration of the extraction window in ns + t_peak: sample corresponding the the highest peak of the trace + window_shift: number of samples before the peak to integrate charge + window_width: duration of the extraction window in samples Returns: masked_wfs: a mask array @@ -150,14 +156,14 @@ def make_masked_array(t_peak, window_shift, window_width): shape=(constants.N_GAINS, constants.N_PIXELS, constants.N_SAMPLES), dtype=bool, ) + + sample_times = np.expand_dims(np.arange(constants.N_SAMPLES), axis=(0, 1)) + t_peak = np.expand_dims(t_peak, axis=-1) + t_signal_start = t_peak - window_shift t_signal_stop = t_peak + window_width - window_shift - for g in range(0, constants.N_GAINS): - for i in range(0, constants.N_PIXELS): - masked_wfs[g][i][ - t_signal_start[0, g, i] : t_signal_stop[0, g, i] - ] = True + masked_wfs = (sample_times >= t_signal_start) & (sample_times < t_signal_stop) return masked_wfs @@ -179,5 +185,8 @@ def finish(self): "amp_int_per_pix_per_event" ].dtype.type(self.__amp_int_per_pix_per_event), FF_coef=FlatFieldContainer.fields["FF_coef"].dtype.type(self.__FF_coef), + bad_pixels=FlatFieldContainer.fields["bad_pixels"].dtype.type( + self.__bad_pixels + ), ) return output From bb2c95b08d3647a4bae62316eaa2e828b210a41d Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Fri, 17 Jan 2025 17:56:03 -0300 Subject: [PATCH 17/19] example of how to run the flat field tool twice, including functions to calculate gain, hi/lo ratio and identify bad pixels --- .../ajardinb/flatfield_example.py | 161 +++++++++++++++++- 1 file changed, 158 insertions(+), 3 deletions(-) diff --git a/src/nectarchain/user_scripts/ajardinb/flatfield_example.py b/src/nectarchain/user_scripts/ajardinb/flatfield_example.py index f0855c30..45c153d6 100644 --- a/src/nectarchain/user_scripts/ajardinb/flatfield_example.py +++ b/src/nectarchain/user_scripts/ajardinb/flatfield_example.py @@ -1,15 +1,143 @@ import os +import matplotlib.pyplot as plt +import numpy as np +from ctapipe_io_nectarcam import constants + from nectarchain.makers.calibration import FlatfieldNectarCAMCalibrationTool # Define the global environment variable NECTARCAMDATA (folder where are the runs) -# os.environ["NECTARCAMDATA"] = "./20231222" +os.environ["NECTARCAMDATA"] = "./20231222" + + +def get_gain(output_from_preFlatFieldComponent): + """ + Calculate the gain using the ratio of the variance and the mean amplitude per pixel + + Args: + output_from_preFlatFieldComponent: output from the preFlatFieldComponent + + Returns: + gain: list of gain for each pixel + """ + + amp_int_per_pix_per_event = ( + output_from_preFlatFieldComponent.amp_int_per_pix_per_event[:, :, :] + ) + amp_int_per_pix_mean = np.mean(amp_int_per_pix_per_event, axis=0) + amp_int_per_pix_var = np.var(amp_int_per_pix_per_event, axis=0) + gain = list( + np.divide( + amp_int_per_pix_var, amp_int_per_pix_mean, where=amp_int_per_pix_mean != 0.0 + ) + ) + return gain + + +def get_hi_lo_ratio(output_from_preFlatFieldComponent): + """ + Calculate the high gain to low gain ratio + + Args: + output_from_preFlatFieldComponent: output from the preFlatFieldComponent + + Returns: + hi_lo_ratio: list of hi/lo ratio for each pixel + """ + + gain = get_gain(output_from_preFlatFieldComponent) + hi_lo_ratio = gain[constants.HIGH_GAIN] / gain[constants.LOW_GAIN] + return hi_lo_ratio + + +def get_bad_pixels(output_from_preFlatFieldComponent): + """ + Identify bad pixels + + Args: + output_from_preFlatFieldComponent: output from the preFlatFieldComponent + + Returns: + all_bad_pix: list of bad pixels + """ + + bad_pix = [] + + n_event = len(output_from_preFlatFieldComponent.FF_coef[:, 0, 0]) + step = 100 + n_step = round(n_event / step) + + hi_lo = get_hi_lo_ratio(output_from_preFlatFieldComponent) + + amp_int_per_pix_per_event = preFlatFieldOutput.amp_int_per_pix_per_event[:, :, :] + mean_amp_int_per_pix = np.mean(amp_int_per_pix_per_event, axis=0) + mean_amp = np.mean(mean_amp_int_per_pix, axis=1) + std_amp = np.std(mean_amp_int_per_pix, axis=1) + + for p in range(0, constants.N_PIXELS): + # pixel with hi/lo ratio to small or to high (+/- 5 times the mean hi/lo ratio) + if (hi_lo[p] < np.mean(hi_lo) - (5 * np.std(hi_lo))) or ( + hi_lo[p] > np.mean(hi_lo) + (5 * np.std(hi_lo)) + ): + bad_pix.append(p) + + amp_int_per_pix_per_event = ( + output_from_preFlatFieldComponent.amp_int_per_pix_per_event[:, :, p] + ) + mean_amp_int_per_pix = np.mean(amp_int_per_pix_per_event, axis=0) + + for G in [constants.HIGH_GAIN, constants.LOW_GAIN]: + # pixels with too low amplitude + if mean_amp_int_per_pix[G] < (mean_amp[G] - 10 * std_amp[G]): + bad_pix.append(p) + + # pixels with unstable flat-field coefficient + FF_coef = output_from_preFlatFieldComponent.FF_coef[:, G, p] + mean_FF_per_pix = np.mean(FF_coef, axis=0) + std_FF_per_pix = np.std(FF_coef, axis=0) + + for e in range(0, round(n_step)): + x_block = np.linspace(e * step, (e + 1) * step, step) + FF_coef_mean_per_block = np.mean( + output_from_preFlatFieldComponent.FF_coef[ + e * step : (e + 1) * step, G, p + ], + axis=0, + ) + FF_coef_std_per_block = np.std( + output_from_preFlatFieldComponent.FF_coef[ + e * step : (e + 1) * step, G, p + ], + axis=0, + ) + + if ( + FF_coef_mean_per_block < mean_FF_per_pix - FF_coef_std_per_block + ) or (FF_coef_mean_per_block > mean_FF_per_pix + FF_coef_std_per_block): + bad_pix.append(p) + + all_bad_pix = list(set(bad_pix)) + all_bad_pix.sort() + + return all_bad_pix + + +# defalut gain array +gain_default = 58.0 +hi_lo_ratio_default = 13.0 +gain_array = list(np.ones(shape=(constants.N_GAINS, constants.N_PIXELS))) +gain_array[0] = gain_array[0] * gain_default +gain_array[1] = gain_array[1] * gain_default / hi_lo_ratio_default + +# empty list of bad pixels +bad_pixels_array = list([]) run_number = 4940 max_events = 10000 -window_width = 14 +window_width = 12 +outfile = os.environ["NECTARCAMDATA"] + "/FlatFieldTests/1FF_{}.h5".format(run_number) -# Call the tool +# Initial call tool = FlatfieldNectarCAMCalibrationTool( progress_bar=True, run_number=run_number, @@ -17,6 +145,9 @@ log_level=20, window_width=window_width, overwrite=True, + gain=gain_array, + bad_pix=bad_pixels_array, + output_path=outfile, ) tool.initialize() @@ -24,3 +155,27 @@ tool.start() preFlatFieldOutput = tool.finish(return_output_component=True)[0] + +outfile = os.environ["NECTARCAMDATA"] + "/FlatFieldTests/2FF_{}.h5".format(run_number) + +# Second call with updated gain aray and bad pixels +tool = FlatfieldNectarCAMCalibrationTool( + progress_bar=True, + run_number=run_number, + max_events=max_events, + log_level=20, + window_width=window_width, + overwrite=True, + gain=get_gain(preFlatFieldOutput), + bad_pix=get_bad_pixels(preFlatFieldOutput), + output_path=outfile, +) + +tool.initialize() +tool.setup() + +tool.start() +FlatFieldOutput = tool.finish(return_output_component=True)[0] + +# Another option would be to use only one tool and make the gain calculation and +# identification of bad pixels in the finish fonction of the component (to be tested) From d877494126dc94d9a671a010d072b99095cd977c Mon Sep 17 00:00:00 2001 From: jlenain Date: Wed, 22 Jan 2025 16:27:00 +0100 Subject: [PATCH 18/19] Resolve conflicts with `main` following merge of PR #168 --- src/nectarchain/makers/component/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/nectarchain/makers/component/__init__.py b/src/nectarchain/makers/component/__init__.py index 3843da23..8bb7ac9a 100644 --- a/src/nectarchain/makers/component/__init__.py +++ b/src/nectarchain/makers/component/__init__.py @@ -12,7 +12,13 @@ from .photostatistic_algorithm import PhotoStatisticAlgorithm from .photostatistic_component import PhotoStatisticNectarCAMComponent from .preflatfield_component import PreFlatFieldComponent -from .spe import SPECombinedalgorithm, SPEHHValgorithm, SPEHHVStdalgorithm +from .spe import ( + SPECombinedalgorithm, + SPEHHValgorithm, + SPEHHVStdalgorithm, + SPEnominalalgorithm, + SPEnominalStdalgorithm, +) from .waveforms_component import WaveformsComponent __all__ = [ @@ -21,6 +27,8 @@ "SPEHHValgorithm", "SPEHHVStdalgorithm", "SPECombinedalgorithm", + "SPEnominalStdalgorithm", + "SPEnominalalgorithm", "FlatFieldSingleHHVSPENectarCAMComponent", "FlatFieldSingleHHVSPEStdNectarCAMComponent", "FlatFieldSingleNominalSPENectarCAMComponent", From cd8eeef33667c5427e7603e345f2f645f1442c7f Mon Sep 17 00:00:00 2001 From: ArmelleJB Date: Wed, 22 Jan 2025 12:42:56 -0300 Subject: [PATCH 19/19] ensure that the masked array is boolean --- src/nectarchain/makers/component/preflatfield_component.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nectarchain/makers/component/preflatfield_component.py b/src/nectarchain/makers/component/preflatfield_component.py index 15189f77..729d7902 100644 --- a/src/nectarchain/makers/component/preflatfield_component.py +++ b/src/nectarchain/makers/component/preflatfield_component.py @@ -104,7 +104,9 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): masked_wfs[:, self.bad_pix, :] = False # get integrated amplitude and mean amplitude over all pixels per event - amp_int_per_pix_per_event = np.sum(wfs_pedsub, axis=-1, where=masked_wfs) + amp_int_per_pix_per_event = np.sum( + wfs_pedsub[0], axis=-1, where=masked_wfs.astype("bool") + ) self.__amp_int_per_pix_per_event.append(amp_int_per_pix_per_event) # --< We could use ctapipe.image.extractor.LocalPeakWindowSum >--