diff --git a/astrodata/core.py b/astrodata/core.py index d55b53811..8ea3208c8 100644 --- a/astrodata/core.py +++ b/astrodata/core.py @@ -321,6 +321,15 @@ def nddata(self): """ return self._nddata[0] if self.is_single else self._nddata + @nddata.setter + @assign_only_single_slice + def nddata(self, new_nddata): + self.data = new_nddata.data + self.unit = new_nddata.unit + self.uncertainty = new_nddata.uncertainty + self.mask = new_nddata.mask + self.wcs = new_nddata.wcs + def table(self): # FIXME: do we need this in addition to .tables ? return self._tables.copy() @@ -444,6 +453,20 @@ def wcs(self): def wcs(self, value): self.nddata.wcs = value + @property + @returns_list + def unit(self): + """ + A list of `astropy.units.Unit` objects (or a single object if this + is a single slice) attached to the science data, for each extension. + """ + return [nd.unit for nd in self._nddata] + + @unit.setter + @assign_only_single_slice + def unit(self, value): + self.nddata.unit = value + def __iter__(self): if self.is_single: yield self @@ -849,7 +872,8 @@ def _process_pixel_plane(self, pixim, name=None, top_level=False, return nd - def _append_array(self, data, name=None, header=None, add_to=None): + def _append_array(self, data, name=None, header=None, add_to=None, + unit=None): if name in {'DQ', 'VAR'}: raise ValueError(f"'{name}' need to be associated to a " f"'{DEFAULT_EXTENSION}' one") @@ -866,21 +890,22 @@ def _append_array(self, data, name=None, header=None, add_to=None): hdu = fits.ImageHDU(data, header=header) hdu.header['EXTNAME'] = hname ret = self._append_imagehdu(hdu, name=hname, header=None, - add_to=None) + add_to=None, unit=unit) else: ret = add_to.meta['other'][name] = data return ret - def _append_imagehdu(self, hdu, name, header, add_to): + def _append_imagehdu(self, hdu, name, header, add_to, unit): if name in {'DQ', 'VAR'} or add_to is not None: - return self._append_array(hdu.data, name=name, add_to=add_to) + return self._append_array(hdu.data, name=name, add_to=add_to, + unit=unit) else: nd = self._process_pixel_plane(hdu, name=name, top_level=True, custom_header=header) - return self._append_nddata(nd, name, add_to=None) + return self._append_nddata(nd, name, add_to=None, unit=unit) - def _append_raw_nddata(self, raw_nddata, name, header, add_to): + def _append_raw_nddata(self, raw_nddata, name, header, add_to, unit): # We want to make sure that the instance we add is whatever we specify # as NDDataObject, instead of the random one that the user may pass top_level = add_to is None @@ -889,9 +914,10 @@ def _append_raw_nddata(self, raw_nddata, name, header, add_to): processed_nddata = self._process_pixel_plane(raw_nddata, top_level=top_level, custom_header=header) - return self._append_nddata(processed_nddata, name=name, add_to=add_to) + return self._append_nddata(processed_nddata, name=name, add_to=add_to, + unit=unit) - def _append_nddata(self, new_nddata, name, add_to): + def _append_nddata(self, new_nddata, name, add_to, unit): # NOTE: This method is only used by others that have constructed NDData # according to our internal format. We don't accept new headers at this # point, and that's why it's missing from the signature. 'name' is @@ -900,6 +926,9 @@ def _append_nddata(self, new_nddata, name, add_to): raise TypeError("You can only append NDData derived instances " "at the top level") + if new_nddata.unit is None: + new_nddata.unit = unit + hd = new_nddata.meta['header'] hname = hd.get('EXTNAME', DEFAULT_EXTENSION) if hname == DEFAULT_EXTENSION: @@ -910,7 +939,7 @@ def _append_nddata(self, new_nddata, name, add_to): return new_nddata - def _append_table(self, new_table, name, header, add_to): + def _append_table(self, new_table, name, header, add_to, unit): tb = _process_table(new_table, name, header) hname = tb.meta['header'].get('EXTNAME') @@ -942,7 +971,7 @@ def find_next_num(tables): add_to.meta['other'][hname] = tb return tb - def _append_astrodata(self, ad, name, header, add_to): + def _append_astrodata(self, ad, name, header, add_to, unit): if not ad.is_single: raise ValueError("Cannot append AstroData instances that are " "not single slices") @@ -954,9 +983,10 @@ def _append_astrodata(self, ad, name, header, add_to): if header is not None: new_nddata.meta['header'] = deepcopy(header) - return self._append_nddata(new_nddata, name=None, add_to=None) + return self._append_nddata(new_nddata, name=None, add_to=None, + unit=unit) - def _append(self, ext, name=None, header=None, add_to=None): + def _append(self, ext, name=None, header=None, add_to=None, unit='adu'): """ Internal method to dispatch to the type specific methods. This is called either by ``.append`` to append on top-level objects only or @@ -972,12 +1002,14 @@ def _append(self, ext, name=None, header=None, add_to=None): for bases, method in dispatcher: if isinstance(ext, bases): - return method(ext, name=name, header=header, add_to=add_to) + return method(ext, name=name, header=header, add_to=add_to, + unit=unit) # Assume that this is an array for a pixel plane - return self._append_array(ext, name=name, header=header, add_to=add_to) + return self._append_array(ext, name=name, header=header, add_to=add_to, + unit=unit) - def append(self, ext, name=None, header=None): + def append(self, ext, name=None, header=None, unit='adu'): """ Adds a new top-level extension. @@ -998,6 +1030,8 @@ def append(self, ext, name=None, header=None): It can consist in a combination of numbers and letters, with the restriction that the letters have to be all capital, and the first character cannot be a number ("[A-Z][A-Z0-9]*"). + header : `astropy.io.fits.Header` + FITS Header to be associated with the NDData or Table object. Returns -------- @@ -1032,7 +1066,7 @@ def append(self, ext, name=None, header=None): UserWarning) name = name.upper() - return self._append(ext, name=name, header=header) + return self._append(ext, name=name, header=header, unit=unit) @classmethod def read(cls, source, extname_parser=None): diff --git a/astrodata/fits.py b/astrodata/fits.py index 5116fd61f..bea2ae431 100644 --- a/astrodata/fits.py +++ b/astrodata/fits.py @@ -30,6 +30,14 @@ NO_DEFAULT = object() LOGGER = logging.getLogger(__name__) +known_invalid_fits_unit_strings = { + 'ELECTRONS/S': u.electron/u.s, + 'ELECTRONS': u.electron, + 'ELECTRON': u.electron, + 'electrons': u.electron, + 'electron': u.electron, +} + class FitsHeaderCollection: """Group access to a list of FITS Header-like objects. @@ -391,7 +399,7 @@ def _prepare_hdulist(hdulist, default_extension='SCI', extname_parser=None): return HDUList(sorted(new_list, key=fits_ext_comp_key)) -def read_fits(cls, source, extname_parser=None): +def read_fits(cls, source, extname_parser=None, default_unit='adu'): """ Takes either a string (with the path to a file) or an HDUList as input, and tries to return a populated AstroData (or descendant) instance. @@ -477,12 +485,24 @@ def associated_extensions(ver): not isinstance(parts['uncertainty'], FitsLazyLoadable)): parts['uncertainty'] = ADVarianceUncertainty(parts['uncertainty']) + bunit = header.get('BUNIT') + if bunit: + if bunit in known_invalid_fits_unit_strings: + bunit = known_invalid_fits_unit_strings[bunit] + try: + unit = u.Unit(bunit, format='fits') + except ValueError: + unit = u.Unit(default_unit) + else: + unit = u.Unit(default_unit) + # Create the NDData object nd = NDDataObject( data=parts['data'], uncertainty=parts['uncertainty'], mask=parts['mask'], meta={'header': header}, + unit=unit, ) if parts['wcs'] is not None: @@ -570,9 +590,17 @@ def ad_to_hdulist(ad): if 'APPROXIMATE' not in wcs_dict.get('FITS-WCS', ''): wcs = None # There's no need to create a WCS extension - hdul.append(new_imagehdu(ext.data, header, 'SCI')) + data_hdu = new_imagehdu(ext.data, header, 'SCI') + + if ext.unit is not None and ext.unit is not u.dimensionless_unscaled: + data_hdu.header['BUNIT'] = (ext.unit.to_string(), + "Physical units of the array values") + + hdul.append(data_hdu) + if ext.uncertainty is not None: hdul.append(new_imagehdu(ext.uncertainty.array, header, 'VAR')) + if ext.mask is not None: hdul.append(new_imagehdu(ext.mask, header, 'DQ')) diff --git a/astrodata/nddata.py b/astrodata/nddata.py index c3ac1d25a..69879a1ee 100644 --- a/astrodata/nddata.py +++ b/astrodata/nddata.py @@ -8,8 +8,8 @@ from copy import deepcopy from functools import reduce +import astropy.units as u import numpy as np - from astropy.io.fits import ImageHDU from astropy.modeling import Model, models from astropy.nddata import (NDArithmeticMixin, NDData, NDSlicingMixin, @@ -369,6 +369,67 @@ def variance(self, value): self.uncertainty = (ADVarianceUncertainty(value) if value is not None else None) + # Override unit so that we can add a setter. + @property + def unit(self): + return self._unit + + @unit.setter + def unit(self, value): + if value is None: + self._unit = None + else: + self._unit = u.Unit(value) + + def convert_unit_to(self, unit, equivalencies=[]): + """ + Returns a new `NDData` object whose values have been converted + to a new unit. + + Copied from Astropy's NDDataArray. + + Parameters + ---------- + unit : `astropy.units.UnitBase` instance or str + The unit to convert to. + + equivalencies : list of equivalence pairs, optional + A list of equivalence pairs to try if the units are not + directly convertible. See :ref:`unit_equivalencies`. + + Returns + ------- + result : `~astropy.nddata.NDData` + The resulting dataset + + Raises + ------ + UnitsError + If units are inconsistent. + + """ + if self.unit is None: + raise ValueError("No unit specified on source data") + data = self.unit.to(unit, self.data, equivalencies=equivalencies) + if self.uncertainty is not None: + uncertainty_values = self.unit.to(unit, self.uncertainty.array, + equivalencies=equivalencies) + # should work for any uncertainty class + uncertainty = self.uncertainty.__class__(uncertainty_values) + else: + uncertainty = None + if self.mask is not None: + new_mask = self.mask.copy() + else: + new_mask = None + # Call __class__ in case we are dealing with an inherited type + result = self.__class__(data, uncertainty=uncertainty, + mask=new_mask, + wcs=self.wcs, + meta=self.meta, unit=unit) + + return result + def set_section(self, section, input): """ Sets only a section of the data. This method is meant to prevent diff --git a/astrodata/tests/test_core.py b/astrodata/tests/test_core.py index ab80e42fc..537049f65 100644 --- a/astrodata/tests/test_core.py +++ b/astrodata/tests/test_core.py @@ -6,6 +6,7 @@ from numpy.testing import assert_array_equal import astrodata +import astropy.units as u from astropy.io import fits from astropy.nddata import NDData, VarianceUncertainty from astropy.table import Table @@ -48,7 +49,7 @@ def test_attributes(ad1): (operator.truediv, 0.5, 2) ]) def test_arithmetic(op, res, res2, ad1, ad2): - for data in (ad2, ad2.data): + for data in (ad2, ad2.data * u.adu): result = op(ad1, data) assert_array_equal(result.data, res) assert isinstance(result, astrodata.AstroData) @@ -56,10 +57,10 @@ def test_arithmetic(op, res, res2, ad1, ad2): assert isinstance(result[0].data, np.ndarray) assert isinstance(result[0].hdr, fits.Header) - result = op(data, ad1) - assert_array_equal(result.data, res2) + result = op(ad2, ad1) + assert_array_equal(result.data, res2) - for data in (ad2[0], ad2[0].data): + for data in (ad2[0], ad2[0].data * u.adu): result = op(ad1[0], data) assert_array_equal(result.data, res) assert isinstance(result, astrodata.AstroData) @@ -67,7 +68,6 @@ def test_arithmetic(op, res, res2, ad1, ad2): assert isinstance(result[0].data, np.ndarray) assert isinstance(result[0].hdr, fits.Header) - # FIXME: should work also with ad2[0].data, but crashes result = op(ad2[0], ad1[0]) assert_array_equal(result.data, res2) @@ -79,7 +79,7 @@ def test_arithmetic(op, res, res2, ad1, ad2): (operator.itruediv, 0.5, 2) ]) def test_arithmetic_inplace(op, res, res2, ad1, ad2): - for data in (ad2, ad2.data): + for data in (ad2, ad2.data * u.adu): ad = deepcopy(ad1) op(ad, data) assert_array_equal(ad.data, res) @@ -88,11 +88,7 @@ def test_arithmetic_inplace(op, res, res2, ad1, ad2): assert isinstance(ad[0].data, np.ndarray) assert isinstance(ad[0].hdr, fits.Header) - # data2 = deepcopy(ad2[0]) - # op(data2, ad1) - # assert_array_equal(data2, res2) - - for data in (ad2[0], ad2[0].data): + for data in (ad2[0], ad2[0].data * u.adu): ad = deepcopy(ad1) op(ad[0], data) assert_array_equal(ad.data, res) @@ -111,13 +107,13 @@ def test_arithmetic_inplace(op, res, res2, ad1, ad2): def test_arithmetic_multiple_ext(op, res, ad1): ad1.append(np.ones(SHAPE, dtype=np.uint16) + 4) - result = op(ad1, 2) + result = op(ad1, 2 * u.adu) assert len(result) == 2 assert_array_equal(result[0].data, res[0]) assert_array_equal(result[1].data, res[1]) for i, ext in enumerate(ad1): - result = op(ext, 2) + result = op(ext, 2 * u.adu) assert len(result) == 1 assert_array_equal(result[0].data, res[i]) @@ -132,7 +128,7 @@ def test_arithmetic_inplace_multiple_ext(op, res, ad1): ad1.append(np.ones(SHAPE, dtype=np.uint16) + 4) ad = deepcopy(ad1) - result = op(ad, 2) + result = op(ad, 2 * u.adu) assert len(result) == 2 assert_array_equal(result[0].data, res[0]) assert_array_equal(result[1].data, res[1]) @@ -141,13 +137,13 @@ def test_arithmetic_inplace_multiple_ext(op, res, ad1): # as it is now independant from its parent for i, ext in enumerate(ad1): ext = deepcopy(ext) - result = op(ext, 2) + result = op(ext, 2 * u.adu) assert len(result) == 1 assert_array_equal(result[0].data, res[i]) # Now work directly on the input object, will creates single ad objects for i, ext in enumerate(ad1): - result = op(ext, 2) + result = op(ext, 2 * u.adu) assert len(result) == 1 assert_array_equal(result.data, res[i]) @@ -157,7 +153,7 @@ def test_arithmetic_inplace_multiple_ext(op, res, ad1): ('multiply', 3, 3), ('divide', 2, 0.5)]) def test_operations(ad1, op, arg, res): - result = getattr(ad1, op)(arg) + result = getattr(ad1, op)(arg * u.adu) assert_array_equal(result.data, res) assert isinstance(result, astrodata.AstroData) assert isinstance(result[0].data, np.ndarray) @@ -170,6 +166,7 @@ def test_operate(): nd = NDData(data=[[1, 2], [3, 4]], uncertainty=VarianceUncertainty(np.ones((2, 2))), mask=np.identity(2), + unit='adu', meta={'header': fits.Header()}) ad.append(nd) @@ -184,6 +181,7 @@ def test_write_and_read(tmpdir, capsys): nd = NDData(data=[[1, 2], [3, 4]], uncertainty=VarianceUncertainty(np.ones((2, 2))), mask=np.identity(2), + unit='adu', meta={'header': fits.Header()}) ad.append(nd) diff --git a/astrodata/tests/test_object_construction.py b/astrodata/tests/test_object_construction.py index 3b464173a..2903cae86 100644 --- a/astrodata/tests/test_object_construction.py +++ b/astrodata/tests/test_object_construction.py @@ -94,16 +94,19 @@ def test_create_invalid(): def test_append_image_hdu(): + shape = (4, 5) ad = astrodata.create(fits.PrimaryHDU()) - ad.append(fits.ImageHDU(data=np.zeros((4, 5)))) - ad.append(fits.ImageHDU(data=np.zeros((4, 5))), name='SCI') + ad.append(fits.ImageHDU(data=np.zeros(shape))) + ad.append(fits.ImageHDU(data=np.zeros(shape)), name='SCI', unit='electron') with pytest.raises(ValueError, match="Arbitrary image extensions can only be added " "in association to a 'SCI'"): - ad.append(fits.ImageHDU(data=np.zeros((4, 5))), name='SCI2') + ad.append(fits.ImageHDU(data=np.zeros(shape)), name='SCI2') assert len(ad) == 2 + assert ad[0].nddata.unit == 'adu' + assert ad[1].nddata.unit == 'electron' def test_append_lowercase_name(): @@ -113,12 +116,28 @@ def test_append_lowercase_name(): ad.append(NDData(np.zeros((4, 5))), name='sci') +def test_append_nddata_and_units(): + shape = (4, 5) + ad = astrodata.create({}) + ad.append(NDData(np.zeros(shape))) + ad.append(NDData(np.zeros(shape), unit='electron')) + ad.append(NDData(np.zeros(shape)), unit='electron') + + assert ad[0].unit == 'adu' + assert ad[1].unit == 'electron' + assert ad[2].unit == 'electron' + assert ad.unit == ['adu', 'electron', 'electron'] + ad[1].unit = 'adu' + assert ad.unit == ['adu', 'adu', 'electron'] + + def test_append_arrays(tmp_path): testfile = tmp_path / 'test.fits' ad = astrodata.create({}) ad.append(np.zeros(10)) ad[0].ARR = np.arange(5) + ad.append(np.zeros(10), unit='electron') with pytest.raises(AttributeError): ad[0].SCI = np.arange(5) @@ -138,7 +157,9 @@ def test_append_arrays(tmp_path): ad.write(testfile) ad = astrodata.open(testfile) - assert len(ad) == 1 + assert len(ad) == 2 + assert ad[0].nddata.unit == 'adu' + assert ad[1].nddata.unit == 'electron' assert ad[0].nddata.meta['header']['EXTNAME'] == 'SCI' assert_array_equal(ad[0].ARR, np.arange(5)) diff --git a/geminidr/core/primitives_preprocess.py b/geminidr/core/primitives_preprocess.py index f6a25b849..104eb8dd6 100644 --- a/geminidr/core/primitives_preprocess.py +++ b/geminidr/core/primitives_preprocess.py @@ -9,11 +9,11 @@ from copy import deepcopy from functools import partial +import astropy.units as u import astrodata import gemini_instruments # noqa import matplotlib.pyplot as plt import numpy as np -from astrodata import NDAstroData from astrodata.provenance import add_provenance from astropy.table import Table from geminidr import PrimitivesBASE @@ -107,11 +107,10 @@ def ADUToElectrons(self, adinputs=None, suffix=None): "the gain".format(ad.filename)) for ext, gain in zip(ad, gain_list): log.stdinfo(f" gain for extension {ext.id} = {gain}") - ext.multiply(gain) + ext.multiply(gain * u.electron / u.adu) # Update the headers of the AstroData Object. The pixel data now # has units of electrons so update the physical units keyword. - ad.hdr.set('BUNIT', 'electron', self.keyword_comments['BUNIT']) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs diff --git a/geminidr/core/primitives_spect.py b/geminidr/core/primitives_spect.py index 201ae73ba..f60091346 100644 --- a/geminidr/core/primitives_spect.py +++ b/geminidr/core/primitives_spect.py @@ -2092,7 +2092,6 @@ def fluxCalibrate(self, adinputs=None, **params): log.stdinfo(f"{ad.filename}: Correcting for airmass of " f"{delta_airmass:5.3f}") - for index, ext in enumerate(ad): ext_std = std[min(index, len_std-1)] extname = f"{ad.filename} extension {ext.id}" @@ -2107,20 +2106,16 @@ def fluxCalibrate(self, adinputs=None, **params): std_physical_unit = (std_flux_unit.physical_unit if isinstance(std_flux_unit, u.LogUnit) else std_flux_unit) - try: - sci_flux_unit = u.Unit(ext.hdr.get('BUNIT')) - except: - sci_flux_unit = None - if not (std_physical_unit is None or sci_flux_unit is None): - unit = sci_flux_unit * std_physical_unit / flux_units + + if not (std_physical_unit is None or ext.unit is None): + unit = ext.unit * std_physical_unit / flux_units if unit.is_equivalent(u.s): log.fullinfo("Dividing {} by exposure time of {} s". format(extname, exptime)) - ext /= exptime - sci_flux_unit /= u.s + ext /= exptime * u.s elif not unit.is_equivalent(u.dimensionless_unscaled): log.warning(f"{extname} has incompatible units ('" - f"{sci_flux_unit}' and '{std_physical_unit}'" + f"{ext.unit}' and '{std_physical_unit}'" "). Cannot flux calibrate") continue else: @@ -2163,14 +2158,12 @@ def fluxCalibrate(self, adinputs=None, **params): else: sens_factor *= 10**(0.4 * delta_airmass * extinction_correction) - final_sens_factor = (sci_flux_unit * sens_factor / pixel_sizes).to( - final_units, equivalencies=u.spectral_density(waves)).value - + sens_factor /= pixel_sizes if ndim == 2 and dispaxis == 0: - ext *= final_sens_factor[:, np.newaxis] - else: - ext *= final_sens_factor - ext.hdr['BUNIT'] = final_units + sens_factor = sens_factor[:, np.newaxis] + + ext.nddata = ext.nddata.multiply(sens_factor).convert_unit_to( + final_units, equivalencies=u.spectral_density(waves)) # Timestamp and update the filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) @@ -3335,8 +3328,8 @@ def conserve_or_interpolate(ext, user_conserve=None, flux_calibrated=False, bool : whether or not to conserve the flux """ ext_str = f"{ext.filename} extension {ext.id}" - ext_unit = ext.hdr["BUNIT"] - if ext_unit in (None, ""): + + if ext.unit is u.dimensionless_unscaled: if user_conserve is None: this_conserve = not flux_calibrated log.stdinfo(f"{ext_str} has no units but " @@ -3349,13 +3342,12 @@ def conserve_or_interpolate(ext, user_conserve=None, flux_calibrated=False, f"been flux calibrated but conserve={user_conserve}") return this_conserve - ext_unit = u.Unit(ext_unit) # Test for units like flux density units_imply_conserve = True for unit1 in ("W", "photon", "electron", "adu"): for unit2 in ("m2", ""): try: - ext_unit.to(u.Unit(f"{unit1} / ({unit2} nm)"), + ext.unit.to(u.Unit(f"{unit1} / ({unit2} nm)"), equivalencies=u.spectral_density(1. * u.m)) except u.UnitConversionError: pass @@ -3365,15 +3357,15 @@ def conserve_or_interpolate(ext, user_conserve=None, flux_calibrated=False, if flux_calibrated and units_imply_conserve: log.warning(f"Possible unit mismatch for {ext_str}. File has been " - f"flux calibrated but units are {ext_unit}") + f"flux calibrated but units are {ext.unit}") if user_conserve is None: this_conserve = units_imply_conserve log.stdinfo(f"Setting conserve={this_conserve} for {ext_str} since " - f"units are {ext_unit}") + f"units are {ext.unit}") else: if user_conserve != units_imply_conserve: log.warning(f"conserve is set to {user_conserve} but the " - f"units of {ext_str} are {ext_unit}") + f"units of {ext_str} are {ext.unit}") this_conserve = user_conserve # but do what we're told return this_conserve diff --git a/geminidr/core/primitives_visualize.py b/geminidr/core/primitives_visualize.py index 122179f16..723a330e6 100644 --- a/geminidr/core/primitives_visualize.py +++ b/geminidr/core/primitives_visualize.py @@ -605,8 +605,6 @@ def plotSpectraForQA(self, adinputs=None, **params): [float(w), float(s)] for w, s in zip(wavelength, stddev)] - _units = ext.hdr["BUNIT"] - center = np.round(ext.hdr["XTRACTED"]) lower = np.round(ext.hdr["XTRACTLO"]) upper = np.round(ext.hdr["XTRACTHI"]) @@ -619,7 +617,7 @@ def plotSpectraForQA(self, adinputs=None, **params): "wavelength_units": w_units, "id": np.round(center + offset), "intensity": _intensity, - "intensity_units": _units, + "intensity_units": ext.unit.to_string(), "slices": _slices, "stddev": _stddev, } diff --git a/geminidr/core/tests/test_preprocess.py b/geminidr/core/tests/test_preprocess.py index d51d448b2..efc0ec465 100644 --- a/geminidr/core/tests/test_preprocess.py +++ b/geminidr/core/tests/test_preprocess.py @@ -1,16 +1,13 @@ -# import os -# from copy import deepcopy - import os import astrodata import gemini_instruments import numpy as np import pytest +from astropy.io import fits from astrodata.testing import download_from_archive from geminidr.core.primitives_preprocess import Preprocess from geminidr.gemini.lookups import DQ_definitions as DQ -# from geminidr.gmos.primitives_gmos_image import GMOSImage from geminidr.niri.primitives_niri_image import NIRIImage from gempy.library.astrotools import cartesian_regions_to_slices from numpy.testing import (assert_almost_equal, assert_array_almost_equal, @@ -309,6 +306,33 @@ def test_fixpixels_multiple_ext(niriprim2): assert_almost_equal(ad[1].data[sy, sx].max(), 60.333, decimal=2) +def test_adu_to_electrons(astrofaker, caplog, tmp_path): + caplog.set_level('INFO') + ad = astrofaker.create("NIRI", "IMAGE") + ad.init_default_extensions() + ad[0].data[:] = 1 + p = NIRIImage([ad]) + + ad = p.ADUToElectrons()[0] + assert ad[0].unit == 'electron' + assert_array_almost_equal(ad[0].data, ad[0].gain()) + assert caplog.messages[2] == ('Converting N20010101S0001.fits from ADU to ' + 'electrons by multiplying by the gain') + caplog.clear() + + ad = p.ADUToElectrons()[0] + assert ad[0].unit == 'electron' + assert_array_almost_equal(ad[0].data, ad[0].gain()) + assert ('No changes will be made to N20010101S0001_ADUToElectrons.fits, ' + 'since it has already been processed') in caplog.messages[2] + + testfile = tmp_path / 'test.fits' + ad.write(testfile) + assert fits.getval(testfile, 'BUNIT', extname='SCI') == 'electron' + ad = astrodata.open(testfile) + assert ad[0].unit == 'electron' + + # TODO @bquint: clean up these tests # @pytest.fixture @@ -357,16 +381,6 @@ def test_fixpixels_multiple_ext(niriprim2): # assert all(ext.mask[ext.OBJMASK == 1] == ext_orig.mask[ext.OBJMASK == 1] | 1) -# @pytest.mark.xfail(reason="Test needs revision", run=False) -# def test_adu_to_electrons(astrofaker): -# ad = astrofaker.create("NIRI", "IMAGE") -# # astrodata.open(os.path.join(TESTDATAPATH, 'NIRI', 'N20070819S0104_dqAdded.fits')) -# p = NIRIImage([ad]) -# ad = p.ADUToElectrons()[0] -# assert ad_compare(ad, os.path.join(TESTDATAPATH, 'NIRI', -# 'N20070819S0104_ADUToElectrons.fits')) - - # @pytest.mark.xfail(reason="Test needs revision", run=False) # def test_associateSky(): # filenames = ['N20070819S{:04d}_flatCorrected.fits'.format(i) diff --git a/geminidr/core/tests/test_spect.py b/geminidr/core/tests/test_spect.py index 39b7d3952..25e964b0b 100644 --- a/geminidr/core/tests/test_spect.py +++ b/geminidr/core/tests/test_spect.py @@ -309,7 +309,7 @@ def test_flux_conservation_consistency(astrofaker, caplog, unit, ad = astrofaker.create("NIRI") ad.init_default_extensions() p = NIRIImage([ad]) # doesn't matter but we need a log object - ad.hdr["BUNIT"] = unit + ad[0].unit = unit conserve = primitives_spect.conserve_or_interpolate(ad[0], user_conserve=user_conserve, flux_calibrated=flux_calibrated, log=p.log) diff --git a/geminidr/core/tests/test_stack.py b/geminidr/core/tests/test_stack.py index 8561e1ea9..7d8828cc4 100644 --- a/geminidr/core/tests/test_stack.py +++ b/geminidr/core/tests/test_stack.py @@ -4,6 +4,7 @@ import numpy as np import pytest +import astropy.units as u from astropy.io import fits from astropy.table import Table from numpy.testing import assert_array_equal @@ -90,7 +91,7 @@ def test_stackframes_refcat_propagation(niri_adinputs): def test_rejmap(niri_adinputs): for i in (2, 3, 4): - niri_adinputs.append(niri_adinputs[0] + i) + niri_adinputs.append(niri_adinputs[0] + i * u.adu) p = NIRIImage(niri_adinputs) p.prepare() diff --git a/geminidr/gmos/tests/spect/test_calculate_sensitivity.py b/geminidr/gmos/tests/spect/test_calculate_sensitivity.py index 0f4e49f3c..a662ad20d 100644 --- a/geminidr/gmos/tests/spect/test_calculate_sensitivity.py +++ b/geminidr/gmos/tests/spect/test_calculate_sensitivity.py @@ -49,13 +49,13 @@ def _create_fake_data(): _ad = astrofaker.create('GMOS-S') _ad.add_extension(hdu, pixel_scale=1.0) + _ad[0].unit = "electron" _ad[0].data = _ad[0].data.ravel() + 1. _ad[0].mask = np.zeros(_ad[0].data.size, dtype=np.uint16) # ToDo Requires mask _ad[0].variance = np.ones_like(_ad[0].data) # ToDo Requires Variance _ad[0].phu.set('OBJECT', "DUMMY") _ad[0].phu.set('EXPTIME', 1.) - _ad[0].hdr.set('BUNIT', "electron") _ad[0].hdr.set('CTYPE1', "Wavelength") _ad[0].hdr.set('CUNIT1', "nm") _ad[0].hdr.set('CRPIX1', 1) @@ -280,4 +280,4 @@ def create_inputs_recipe(): if "--create-inputs" in sys.argv[1:]: create_inputs_recipe() else: - pytest.main() \ No newline at end of file + pytest.main() diff --git a/geminidr/gmos/tests/spect/test_flux_calibration.py b/geminidr/gmos/tests/spect/test_flux_calibration.py index f49612d92..e5853585f 100644 --- a/geminidr/gmos/tests/spect/test_flux_calibration.py +++ b/geminidr/gmos/tests/spect/test_flux_calibration.py @@ -67,7 +67,6 @@ def _get_spectrophotometric_data(object_name): return wavelength, flux def _create_fake_data(object_name): - from astropy.table import Table astrofaker = pytest.importorskip('astrofaker') wavelength, flux = _get_spectrophotometric_data(object_name) @@ -89,6 +88,7 @@ def _create_fake_data(object_name): _ad = astrofaker.create('GMOS-S') _ad.add_extension(hdu, pixel_scale=1.0) + _ad[0].unit = "electron" _ad[0].data = _ad[0].data.ravel() _ad[0].mask = np.zeros(_ad[0].data.size, dtype=np.uint16) # ToDo Requires mask _ad[0].variance = np.ones_like(_ad[0].data) # ToDo Requires Variance @@ -102,7 +102,6 @@ def _create_fake_data(object_name): _ad[0].hdr.set('NAXIS', 1) _ad[0].phu.set('OBJECT', object_name) _ad[0].phu.set('EXPTIME', 1.) - _ad[0].hdr.set('BUNIT', "electron") assert _ad.object() == object_name assert _ad.exposure_time() == 1 @@ -120,7 +119,6 @@ def _create_fake_data(object_name): @pytest.mark.gmosls @pytest.mark.preprocessed_data -@pytest.mark.regression @pytest.mark.parametrize("ad", test_datasets, indirect=True) def test_regression_on_flux_calibration(ad, ref_ad_factory, change_working_dir): """ @@ -286,4 +284,4 @@ def create_inputs_recipe(): if "--create-inputs" in sys.argv[1:]: create_inputs_recipe() else: - pytest.main() \ No newline at end of file + pytest.main() diff --git a/gempy/library/spectral.py b/gempy/library/spectral.py index 362a494f8..10b5ddc10 100644 --- a/gempy/library/spectral.py +++ b/gempy/library/spectral.py @@ -41,15 +41,9 @@ def __init__(self, spectrum=None, spectral_axis=None, wcs=None, **kwargs): raise TypeError("Input spectrum must be a single AstroData slice") # Unit handling - try: # for NDData-like - flux_unit = spectrum.unit - except AttributeError: - try: # for AstroData - flux_unit = u.Unit(spectrum.hdr.get('BUNIT')) - except (TypeError, ValueError): # unknown/missing - flux_unit = None - if flux_unit is None: - flux_unit = u.dimensionless_unscaled + if spectrum.unit is None: + spectrum.unit = u.dimensionless_unscaled + try: kwargs['mask'] = spectrum.mask except AttributeError: @@ -61,7 +55,7 @@ def __init__(self, spectrum=None, spectral_axis=None, wcs=None, **kwargs): # If spectrum was a Quantity, it already has units so we'd better # not multiply them in again! if not isinstance(flux, u.Quantity): - flux *= flux_unit + flux *= spectrum.unit # If no wavelength information is included, get it from the input if spectral_axis is None and wcs is None: diff --git a/gempy/library/tests/test_spectral.py b/gempy/library/tests/test_spectral.py index a26128739..a07e8b7ab 100644 --- a/gempy/library/tests/test_spectral.py +++ b/gempy/library/tests/test_spectral.py @@ -33,13 +33,13 @@ def _create_fake_data(): _ad.add_extension(hdu, pixel_scale=1.0) _ad[0].wcs = None # or else imaging WCS will be added + _ad[0].unit = units.electron _ad[0].data = _ad[0].data.ravel() + 1. _ad[0].mask = np.zeros(_ad[0].data.size, dtype=np.uint16) # ToDo Requires mask _ad[0].variance = np.ones_like(_ad[0].data) # ToDo Requires Variance _ad[0].phu.set('OBJECT', "DUMMY") _ad[0].phu.set('EXPTIME', 1.) - _ad[0].hdr.set('BUNIT', "electron") _ad[0].hdr.set('CTYPE1', "Wavelength") _ad[0].hdr.set('CUNIT1', "nm") _ad[0].hdr.set('CRPIX1', 1)