ADConversion effect

scopesim/effects/electronic/__init__.py

@@ -20,7 +20,7 @@
 from ...utils import get_logger
 logger = get_logger(__name__)
 
-from .electrons import LinearityCurve, Quantization
+from .electrons import LinearityCurve, ADConversion
 from .noise import (Bias, PoorMansHxRGReadoutNoise, BasicReadoutNoise,
                     ShotNoise, DarkCurrent)
 from .exposure import AutoExposure, SummedExposure

scopesim/effects/electronic/dmps.py

@@ -71,6 +71,7 @@
         self.mode_properties = kwargs["mode_properties"]
 
     def apply_to(self, obj, **kwargs):
+        print("Entering DetectorModePropertiesSetter.apply_to")
         mode_name = kwargs.get("detector_readout_mode",
                                from_currsys("!OBS.detector_readout_mode",
                                             self.cmds))
@@ -83,6 +84,7 @@
         self.cmds["!OBS.detector_readout_mode"] = mode_name
         for key, value in props_dict.items():
             self.cmds[key] = value
+            print(key, value)
 
         return obj
 

scopesim/effects/electronic/electrons.py

@@ -2,7 +2,7 @@
 """Effects related to the conversion of photons into electrons.
 
    - LinearityCurve: Detector linearity
-   - Quantization:   Conversion from electrons to ADU
+   - ADConversion:   Conversion from electrons to ADU
 
 Related effects:
    - QuantumEfficiencyCurve: can be found in ter_curves.py
@@ -12,7 +12,8 @@
 
 from .. import Effect
 from ...base_classes import DetectorBase
-from ...utils import from_currsys, figure_factory, check_keys
+from ...utils import figure_factory, check_keys
+from ...utils import from_currsys, real_colname
 from . import logger
 
 
@@ -89,8 +90,12 @@
         return fig
 
 
-class Quantization(Effect):
-    """Converts raw data to whole photons."""
+class ADConversion(Effect):
+    """Analogue-Digital Conversion effect.
+
+    The effect applies the gain to convert from electrons to ADU
+    and converts the output to the desired data type (e.g. uint16).
+    """
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -102,46 +107,65 @@
         self.meta.update(kwargs)
 
     def _should_apply(self) -> bool:
+        """Check cases where the effect should not be applied"""
         if self.cmds is None:
-            logger.warning("Cannot access cmds for Quantization effect.")
+            logger.warning("Cannot access cmds for ADConversion effect.")
             return True
 
+        # ..todo: need to deal with this case more realistically
         if self.cmds.get("!OBS.autoexpset", False):
             logger.debug("DIT, NDIT determined by AutoExposure -> "
-                         "quantization is not applied.")
+                         "ADConversion is not applied.")
             return False
 
         if self.cmds["!OBS.ndit"] > 1:
-            logger.debug("NDIT set to 1 -> quantization is not applied.")
+            logger.debug("NDIT set to 1 -> digitization is not applied.")
             return False
 
         return True
 
     def apply_to(self, obj, **kwargs):
+        print("Entering ADConversion.apply_to")
         if not isinstance(obj, DetectorBase):
             return obj
 
         if not self._should_apply():
             return obj
 
+        print("Applying ADConversion")
         new_dtype = self.meta["dtype"]
         if not np.issubdtype(new_dtype, np.integer):
-            logger.warning("Setting quantized data to dtype %s, which is not "
+            logger.warning("Setting digitized data to dtype %s, which is not "
                            "an integer subtype.", new_dtype)
 
+        # TODO: Apply the gain value (copy from DarkCurrent)
+        print(from_currsys(self.meta['gain'], self.cmds))
+        if isinstance(from_currsys(self.meta["gain"], self.cmds), dict):
+            dtcr_id = obj.meta[real_colname("id", obj.meta)]
+            gain = from_currsys(self.meta["gain"][dtcr_id], self.cmds)
+        elif isinstance(from_currsys(self.meta["gain"], self.cmds), (float, int)):
+            gain = from_currsys(self.meta["gain"], self.cmds)
+        else:
+            raise ValueError("<ADConversion>.meta['gain'] must be either "
+                             f"dict or float, but is {self.meta['gain']}")
+
+
+        # Apply gain   TODO: option to turn this off
+        obj._hdu.data /= gain
+
         # Remove values below 0, because for unsigned types these get wrapped
         # around to MAXINT, which is even worse than negative values.
         # TODO: Write a test for this.
         if np.issubdtype(new_dtype, np.unsignedinteger):
             negvals_mask = obj._hdu.data < 0
             if negvals_mask.any():
-                logger.warning(f"Effect Quantization: {negvals_mask.sum()} negative pixels")
+                logger.warning(f"Effect ADConversion: {negvals_mask.sum()} negative pixels")
                 obj._hdu.data[negvals_mask] = 0
 
         # This used to create a new ImageHDU with the same header but the data
         # set to the modified data. It should be fine to simply re-assign the
         # data attribute, but just in case it's not...
-        logger.debug("Applying quantization to dtype %s.", new_dtype)
+        logger.debug("Applying digitization to dtype %s.", new_dtype)
         obj._hdu.data = np.floor(obj._hdu.data).astype(new_dtype)
 
         return obj

scopesim/tests/test_basic_instrument/test_basic_instrument.py

@@ -1,4 +1,4 @@
-
+"""Test a basic instrument setup"""
 import pytest
 
 import numpy as np
@@ -8,7 +8,7 @@
 
 import scopesim as sim
 from scopesim.source import source_templates as st
-from scopesim.effects import AutoExposure, Quantization
+from scopesim.effects import AutoExposure, ADConversion
 
 
 PLOTS = False
@@ -23,6 +23,8 @@
     "config_fits_keywords",
 ]
 
+# pylint: disable=missing-class-docstring
+# pylint: disable=missing-function-docstring
 
 @pytest.mark.usefixtures("protect_currsys", "patch_all_mock_paths")
 class TestLoadsUserCommands:
@@ -209,13 +211,13 @@ def basic_opt_observed():
     opt.observe(src)
     default = int(opt.readout()[0][1].data.sum())
 
-    quanteff = Quantization(cmds=opt.cmds)
-    opt.optics_manager["basic_detector"].effects.append(quanteff)
-    return opt, default, quanteff
+    adconverter = ADConversion(cmds=opt.cmds)
+    opt.optics_manager["basic_detector"].effects.append(adconverter)
+    return opt, default, adconverter
 
 
 @pytest.fixture(scope="function", name="obs_aeq")
-def basic_opt_with_autoexp_and_quant_observed():
+def basic_opt_with_autoexp_and_digitize_observed():
     src = st.star(flux=15)
     cmd = sim.UserCommands(use_instrument="basic_instrument",
                            ignore_effects=SWITCHOFF,
@@ -226,112 +228,112 @@ def basic_opt_with_autoexp_and_quant_observed():
 
     autoexp = AutoExposure(cmds=opt.cmds, mindit=1,
                            full_well=100, fill_frac=0.8)
-    quanteff = Quantization(cmds=opt.cmds)
+    adconverter = ADConversion(cmds=opt.cmds)
     opt.optics_manager["basic_detector"].effects.insert(2, autoexp)
-    opt.optics_manager["basic_detector"].effects.append(quanteff)
+    opt.optics_manager["basic_detector"].effects.append(adconverter)
     opt.cmds["!OBS.exptime"] = 60
-    return opt, default, quanteff
+    return opt, default, adconverter
 
 
 @pytest.mark.usefixtures("protect_currsys", "patch_all_mock_paths")
 class TestDitNdit:
-    @pytest.mark.parametrize(("dit", "ndit", "factor", "quant"),
+    @pytest.mark.parametrize(("dit", "ndit", "factor", "adconvert"),
                              [(20, 1, 2, True), (10, 3, 3, False)])
-    def test_obs_dict(self, obs, dit, ndit, factor, quant):
+    def test_obs_dict(self, obs, dit, ndit, factor, adconvert):
         """This should just use dit, ndit from !OBS."""
-        opt, default, quanteff = obs
+        opt, default, adconverter = obs
         # This should work with patch.dict, but doesn't :(
         o_dit, o_ndit = opt.cmds["!OBS.dit"], opt.cmds["!OBS.ndit"]
         opt.cmds["!OBS.dit"] = dit
         opt.cmds["!OBS.ndit"] = ndit
         kwarged = int(opt.readout()[0][1].data.sum())
-        assert quanteff._should_apply() == quant
+        assert adconverter._should_apply() == adconvert
         opt.cmds["!OBS.dit"] = o_dit
         opt.cmds["!OBS.ndit"] = o_ndit
-        # Quantization results in ~4% loss, which is fine:
+        # Digitization results in ~4% loss, which is fine:
         assert pytest.approx(kwarged / default, rel=.05) == factor
 
-    @pytest.mark.parametrize(("dit", "ndit", "factor", "quant"),
+    @pytest.mark.parametrize(("dit", "ndit", "factor", "adconvert"),
                              [pytest.param(20, 1, 2, True, marks=pytest.mark.xfail(reason="S.E. doesn't get kwargs without A.E..")),
                               pytest.param(10, 3, 3, False, marks=pytest.mark.xfail(reason="S.E. doesn't get kwargs without A.E..")),
                               (None, None, 1, True)])
-    def test_kwargs_override_obs_dict(self, obs, dit, ndit, factor, quant):
+    def test_kwargs_override_obs_dict(self, obs, dit, ndit, factor, adconvert):
         """This should prioritize kwargs and fallback to !OBS."""
-        opt, default, quanteff = obs
+        opt, default, adconverter = obs
         kwarged = int(opt.readout(dit=dit, ndit=ndit)[0][1].data.sum())
-        assert quanteff._should_apply() == quant
-        # Quantization results in ~4% loss, which is fine:
+        assert adconverter._should_apply() == adconvert
+        # Digitization results in ~4% loss, which is fine:
         assert pytest.approx(kwarged / default, rel=.05) == factor
 
-    @pytest.mark.parametrize(("dit", "ndit", "factor", "quant"),
+    @pytest.mark.parametrize(("dit", "ndit", "factor", "adconvert"),
                              [pytest.param(20, 1, 2, True, marks=pytest.mark.xfail(reason="S.E. doesn't get kwargs without A.E..")),
                               pytest.param(10, 3, 3, False, marks=pytest.mark.xfail(reason="S.E. doesn't get kwargs without A.E..")),
                               pytest.param(None, None, 1, True, marks=pytest.mark.xfail(reason="A.E. dosen't use dit, ndit from !OBS if those are None in kwargs."))])
     def test_kwargs_override_obs_dict_also_with_autoexp(
-            self, obs_aeq, dit, ndit, factor, quant):
+            self, obs_aeq, dit, ndit, factor, adconvert):
         """This should prioritize dit, ndit from kwargs.
 
         Lacking those, dit and ndit from !OBS should be used over exptime.
         """
-        opt, default, quanteff = obs_aeq
+        opt, default, adconverter = obs_aeq
         kwarged = int(opt.readout(dit=dit, ndit=ndit)[0][1].data.sum())
         assert int(kwarged / default) == factor
-        assert quanteff._should_apply() == quant
+        assert adconverter._should_apply() == adconvert
 
     @pytest.mark.parametrize(("exptime", "factor"),
                              [(20, 2), (30, 3),
                               pytest.param(None, 6, marks=pytest.mark.xfail(reason="A.E. doesn't understand None yet."))])
     def test_autoexp(self, obs_aeq, exptime, factor):
         """This should prioritize kwargs and fallback to !OBS."""
-        opt, default, quanteff = obs_aeq
+        opt, default, adconverter = obs_aeq
         # This should work with patch.dict, but doesn't :(
         o_dit, o_ndit = opt.cmds["!OBS.dit"], opt.cmds["!OBS.ndit"]
         opt.cmds["!OBS.dit"] = None
         opt.cmds["!OBS.ndit"] = None
         kwarged = int(opt.readout(exptime=exptime)[0][1].data.sum())
-        assert not quanteff._should_apply()
+        assert not adconverter._should_apply()
         opt.cmds["!OBS.dit"] = o_dit
         opt.cmds["!OBS.ndit"] = o_ndit
         assert int(kwarged / default) == factor
 
-    @pytest.mark.parametrize(("exptime", "factor", "quant"),
+    @pytest.mark.parametrize(("exptime", "factor", "adconvert"),
                              [(30, 3, False),
                               (None, 1, True)])
-    def test_autoexp_overrides_obs_dict(self, obs_aeq, exptime, factor, quant):
+    def test_autoexp_overrides_obs_dict(self, obs_aeq, exptime, factor, adconvert):
         """This should prioritize kwargs and use dit, ndit when None."""
-        opt, default, quanteff = obs_aeq
+        opt, default, adconverter = obs_aeq
         kwarged = int(opt.readout(exptime=exptime)[0][1].data.sum())
-        assert quanteff._should_apply() == quant
-        # Quantization results in ~4% loss, which is fine:
+        assert adconverter._should_apply() == adconvert
+        # Digitization results in ~4% loss, which is fine:
         assert pytest.approx(kwarged / default, rel=.05) == factor
 
-    @pytest.mark.parametrize(("dit", "ndit", "factor", "quant"),
+    @pytest.mark.parametrize(("dit", "ndit", "factor", "adconvert"),
                              [pytest.param(90, 1, 90, True, marks=pytest.mark.xfail(reason="S.E. doesn't get kwargs.")),
                               pytest.param(2, 90, 180, False, marks=pytest.mark.xfail(reason="S.E. doesn't get kwargs."))])
     def test_ditndit_in_kwargs_while_also_having_autoexp(
-            self, obs_aeq, dit, ndit, factor, quant):
+            self, obs_aeq, dit, ndit, factor, adconvert):
         """This should prioritize dit, ndit from kwargs."""
-        opt, default, quanteff = obs_aeq
+        opt, default, adconverter = obs_aeq
         kwarged = int(opt.readout(dit=dit, ndit=ndit)[0][1].data.sum())
         assert int(kwarged / default) == factor
-        assert quanteff._should_apply() == quant
+        assert adconverter._should_apply() == adconvert
 
-    @pytest.mark.parametrize(("dit", "ndit", "exptime", "factor", "quant"),
+    @pytest.mark.parametrize(("dit", "ndit", "exptime", "factor", "adconvert"),
                              [pytest.param(90, 1, None, 90, True, marks=pytest.mark.xfail(reason="S.E. doesn't get kwargs.")),
                               pytest.param(2, 90, 20, 180, False, marks=pytest.mark.xfail(reason="S.E. doesn't get kwargs."))])
     def test_ditndit_in_kwargs_while_also_having_autoexp_and_exptime(
-            self, obs_aeq, dit, ndit, exptime, factor, quant):
+            self, obs_aeq, dit, ndit, exptime, factor, adconvert):
         """This should prioritize dit, ndit from kwargs and ignore exptime."""
-        opt, default, quanteff = obs_aeq
+        opt, default, adconverter = obs_aeq
         kwarged = int(opt.readout(exptime=exptime,
                                   dit=dit, ndit=ndit)[0][1].data.sum())
         assert int(kwarged / default) == factor
-        assert quanteff._should_apply() == quant
+        assert adconverter._should_apply() == adconvert
 
     @pytest.mark.xfail(reason="Currently raises TypeError, but should be ValueError.")
     def test_throws_for_no_anything(self, obs):
         """No specification whatsoever, so throw error."""
-        opt, default, quanteff = obs
+        opt, default, adconverter = obs
         opt.cmds["!OBS.exptime"] = None
         # This should work with patch.dict, but doesn't :(
         o_dit, o_ndit = opt.cmds["!OBS.dit"], opt.cmds["!OBS.ndit"]
@@ -344,15 +346,15 @@ def test_throws_for_no_anything(self, obs):
 
     def test_throws_for_no_ditndit_no_autoexp_kwargs(self, obs):
         """This should use exptime from kwargs, but fail w/o AutoExp."""
-        opt, default, quanteff = obs
+        opt, default, adconverter = obs
         opt.cmds["!OBS.exptime"] = None
         with pytest.raises(ValueError):
             opt.readout(exptime=60)
 
     @pytest.mark.xfail(reason="Currently raises TypeError, but should be ValueError.")
     def test_throws_for_no_ditndit_no_autoexp_obs(self, obs):
         """This should fallback to !OBS.exptime, but fail w/o AutoExp."""
-        opt, default, quanteff = obs
+        opt, default, adconverter = obs
         opt.cmds["!OBS.exptime"] = 60
         # This should work with patch.dict, but doesn't :(
         o_dit, o_ndit = opt.cmds["!OBS.dit"], opt.cmds["!OBS.ndit"]