diff --git a/CHANGES.rst b/CHANGES.rst index a73ccc3f3..a812e498f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,9 @@ New Features ^^^^^^^^^^^^ +- Added new SDSS-V ``Spectrum1D`` and ``SpectrumList`` default loaders for + ``astraStar`` and ``astraVisit`` model spectra datatypes. [#1203] + Bug Fixes ^^^^^^^^^ @@ -47,15 +50,23 @@ Bug Fixes - Fixed ``Spectrum1D.with_flux_unit()`` not converting uncertainty along with flux unit. [#1181] +- Fixed ``mwmVisit`` SDSS-V ``Spectrum1D`` and ``SpectrumList`` default loader + being unable to load files containing only BOSS instrument spectra. [#1185] + +- Fixed automatic format detection for SDSS-V ``SpectrumList`` default loaders. [#1185] + - Fixed extracting a spectral region when one of spectrum/region is in wavelength and the other is in frequency units. [#1187] + Other Changes and Additions ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Replaced ``LevMarLSQFitter`` with ``TRFLSQFitter`` as the former is no longer recommended by ``astropy``. [#1180] +- "Multi" loaders have been removed from SDSS-V ``SpectrumList`` default loaders. [#1185] + 1.17.0 (2024-10-04) ------------------- diff --git a/specutils/io/default_loaders/sdss_v.py b/specutils/io/default_loaders/sdss_v.py index ab5f92d45..7693fb36c 100644 --- a/specutils/io/default_loaders/sdss_v.py +++ b/specutils/io/default_loaders/sdss_v.py @@ -1,4 +1,5 @@ """Register reader functions for various spectral formats.""" + import warnings from typing import Optional @@ -84,7 +85,22 @@ def mwm_identify(origin, *args, **kwargs): with read_fileobj_or_hdulist(*args, **kwargs) as hdulist: return (("V_ASTRA" in hdulist[0].header.keys()) and len(hdulist) > 0 and ("SDSS_ID" in hdulist[0].header.keys()) - and (isinstance(hdulist[i], BinTableHDU) for i in range(1, 5))) + and (isinstance(hdulist[i], BinTableHDU) for i in range(1, 5)) + and all("model_flux" not in hdulist[i].columns.names + for i in range(1, 5))) + + +def astra_identify(origin, *args, **kwargs): + """ + Check whether given input is FITS and has SDSS-V astra model spectra + BINTABLE in all 4 extensions. This is used for Astropy I/O Registry. + """ + with read_fileobj_or_hdulist(*args, **kwargs) as hdulist: + return (("V_ASTRA" in hdulist[0].header.keys()) and len(hdulist) > 0 + and ("SDSS_ID" in hdulist[0].header.keys()) + and (isinstance(hdulist[i], BinTableHDU) for i in range(1, 5)) + and all("model_flux" in hdulist[i].columns.names + for i in range(1, 5))) def _wcs_log_linear(naxis, cdelt, crval): @@ -340,7 +356,7 @@ def load_sdss_spec_1D(file_obj, *args, hdu: Optional[int] = None, **kwargs): """ if hdu is None: # TODO: how should we handle this -- multiple things in file, but the user cannot choose. - warnings.warn('HDU not specified. Loading coadd spectrum (HDU1)', + warnings.warn("HDU not specified. Loading coadd spectrum (HDU1)", AstropyUserWarning) hdu = 1 # defaulting to coadd # raise ValueError("HDU not specified! Please specify a HDU to load.") @@ -468,14 +484,15 @@ def load_sdss_mwm_1d(file_obj, if (np.array(datasums) == 0).all(): raise ValueError("Specified file is empty.") - # TODO: how should we handle this -- multiple things in file, but the user cannot choose. if hdu is None: for i in range(1, len(hdulist)): if hdulist[i].header.get("DATASUM") != "0": hdu = i warnings.warn( - 'HDU not specified. Loading spectrum at (HDU{})'. - format(i), AstropyUserWarning) + "HDU not specified. Loading spectrum at (HDU{})". + format(i), + AstropyUserWarning, + ) break # load spectra and return @@ -594,7 +611,7 @@ def _load_mwmVisit_or_mwmStar_hdu(hdulist: HDUList, hdu: int, **kwargs): # choose between mwmVisit/Star via KeyError except try: - meta['mjd'] = hdulist[hdu].data['mjd'] + meta["mjd"] = hdulist[hdu].data["mjd"] meta["datatype"] = "mwmVisit" except KeyError: meta["min_mjd"] = str(hdulist[hdu].data["min_mjd"][0]) @@ -602,7 +619,182 @@ def _load_mwmVisit_or_mwmStar_hdu(hdulist: HDUList, hdu: int, **kwargs): meta["datatype"] = "mwmStar" finally: meta["name"] = hdulist[hdu].name - meta["sdss_id"] = hdulist[hdu].data['sdss_id'] + meta["sdss_id"] = hdulist[hdu].data["sdss_id"] + + # drop back a list of Spectrum1Ds to unpack + return Spectrum1D( + spectral_axis=spectral_axis, + flux=flux, + uncertainty=e_flux, + mask=mask, + meta=meta, + ) + + +@data_loader( + "SDSS-V astra model", + identifier=astra_identify, + dtype=Spectrum1D, + priority=20, + extensions=["fits"], +) +def load_astra_1d(file_obj, hdu: Optional[int] = None, **kwargs): + """ + Load an astra model spectrum file as a Spectrum1D. + + Parameters + ---------- + file_obj : str, file-like, or HDUList + FITS file name, file object, or HDUList. + hdu : int + Specified HDU to load. + + Returns + ------- + spectrum : Spectrum1D + The spectra contained in the file from the provided HDU OR the first entry. + """ + with read_fileobj_or_hdulist(file_obj, memmap=False, **kwargs) as hdulist: + # Check if file is empty first + datasums = [] + for i in range(1, len(hdulist)): + datasums.append(int(hdulist[i].header.get("DATASUM"))) + if (np.array(datasums) == 0).all(): + raise ValueError("Specified file is empty.") + + if hdu is None: + for i in range(1, len(hdulist)): + if hdulist[i].header.get("DATASUM") != "0": + hdu = i + warnings.warn( + "HDU not specified. Loading spectrum at (HDU{})". + format(i), + AstropyUserWarning, + ) + break + + return _load_astra_hdu(hdulist, hdu, **kwargs) + + +@data_loader( + "SDSS-V astra model", + identifier=astra_identify, + force=True, + dtype=SpectrumList, + priority=20, + extensions=["fits"], +) +def load_astra_list(file_obj, **kwargs): + """ + Load an astra model spectrum file as a SpectrumList. + + Parameters + ---------- + file_obj : str, file-like, or HDUList + FITS file name, file object, or HDUList. + + Returns + ------- + spectra: SpectrumList + All of the spectra contained in the file. + """ + spectra = SpectrumList() + with read_fileobj_or_hdulist(file_obj, memmap=False, **kwargs) as hdulist: + # Check if file is empty first + datasums = [] + for hdu in range(1, len(hdulist)): + datasums.append(int(hdulist[hdu].header.get("DATASUM"))) + if (np.array(datasums) == 0).all(): + raise ValueError("Specified file is empty.") + + # Now load file + for hdu in range(1, len(hdulist)): + if hdulist[hdu].header.get("DATASUM") == "0": + # Skip zero data HDU's + continue + spectra.append(_load_astra_hdu(hdulist, hdu)) + return spectra + + +def _load_astra_hdu(hdulist: HDUList, hdu: int, **kwargs): + """ + HDU loader subfunction for astra model spectrum files + + Parameters + ---------- + hdulist: HDUList + HDUList generated from imported file. + hdu: int + Specified HDU to load. + + Returns + ------- + Spectrum1D + The spectrum with nD flux contained in the HDU. + + """ + if hdulist[hdu].header.get("DATASUM") == "0": + raise IndexError( + "Attemped to load an empty HDU specified at HDU{}".format(hdu)) + + # Fetch wavelength + # encoded as WCS for visit, and 'wavelength' for star + try: + wavelength = np.array(hdulist[hdu].data["wavelength"])[0] + except KeyError: + wavelength = _wcs_log_linear( + hdulist[hdu].header.get("NPIXELS"), + hdulist[hdu].header.get("CDELT"), + hdulist[hdu].header.get("CRVAL"), + ) + finally: + if wavelength is None: + raise ValueError( + "Couldn't find wavelength data in HDU{}.".format(hdu)) + spectral_axis = Quantity(wavelength, unit=Angstrom) + + # Fetch flux, e_flux + # NOTE:: flux info is not written + flux_unit = Unit("1e-17 erg / (Angstrom cm2 s)") # NOTE: hardcoded unit + flux = Quantity(hdulist[hdu].data["model_flux"] * + hdulist[hdu].data["continuum"], + unit=flux_unit) + e_flux = InverseVariance(array=hdulist[hdu].data["ivar"]) + + # Collect bitmask + mask = hdulist[hdu].data["pixel_flags"] + # NOTE: specutils considers 0/False as valid values, simlar to numpy convention + mask = mask != 0 + + # collapse shape if 1D spectra in 2D array, makes readout easier + if flux.shape[0] == 1: + flux = np.ravel(flux) + e_flux = e_flux[0] # different class + mask = np.ravel(mask) + + # Create metadata + meta = dict() + meta["header"] = hdulist[0].header + + # Add identifiers (obj, telescope, mjd, datatype) + meta["telescope"] = hdulist[hdu].data["telescope"] + meta["instrument"] = hdulist[hdu].header.get("INSTRMNT") + try: # get obj if exists + meta["obj"] = hdulist[hdu].data["obj"] + except KeyError: + pass + + # choose between mwmVisit/Star via KeyError except + try: + meta["mjd"] = hdulist[hdu].data["mjd"] + meta["datatype"] = "astraVisit" + except KeyError: + meta["min_mjd"] = str(hdulist[hdu].data["min_mjd"][0]) + meta["max_mjd"] = str(hdulist[hdu].data["max_mjd"][0]) + meta["datatype"] = "astraStar" + finally: + meta["name"] = hdulist[hdu].name + meta["sdss_id"] = hdulist[hdu].data["sdss_id"] # drop back a list of Spectrum1Ds to unpack return Spectrum1D( diff --git a/specutils/io/default_loaders/tests/test_sdss_v.py b/specutils/io/default_loaders/tests/test_sdss_v.py index fae2bf1ff..9d1225852 100644 --- a/specutils/io/default_loaders/tests/test_sdss_v.py +++ b/specutils/io/default_loaders/tests/test_sdss_v.py @@ -13,7 +13,8 @@ def generate_apogee_hdu(observatory="APO", with_wl=True, datasum="0", - nvisits=1): + nvisits=1, + astra=False): wl = (10**(4.179 + 6e-6 * np.arange(8575))).reshape((1, -1)) flux = np.array([np.zeros_like(wl)] * nvisits) ivar = np.array([np.zeros_like(wl)] * nvisits) @@ -23,15 +24,15 @@ def generate_apogee_hdu(observatory="APO", columns = [ fits.Column(name="spectrum_pk_id", array=[159783564], format="K"), - fits.Column(name="release", array=[b"sdss5"], format="5A"), - fits.Column(name="filetype", array=[b"apStar"], format="6A"), - fits.Column(name="v_astra", array=[b"0.5.0"], format="5A"), + fits.Column(name="release", array=[b"sdss5"] * nvisits, format="5A"), + fits.Column(name="v_astra", array=[b"0.5.0"] * nvisits, format="5A"), fits.Column(name="healpix", array=[3], format="J"), - fits.Column(name="sdss_id", array=[42], format="K"), - fits.Column(name="apred", array=[b"1.2"], format="3A"), + fits.Column(name="sdss_id", array=[42] * nvisits, format="K"), + fits.Column(name="apred", array=[b"1.2"] * nvisits, format="3A"), fits.Column(name="obj", array=[b"2M19534321+6705175"], format="18A"), - fits.Column(name="telescope", array=[b"apo25m"], format="6A"), - fits.Column(name="snr", array=[50], format="E"), + fits.Column(name="telescope", array=[b"apo25m"] * nvisits, + format="6A"), + fits.Column(name="snr", array=[50] * nvisits, format="E"), ] if with_wl: columns.append( @@ -47,8 +48,10 @@ def generate_apogee_hdu(observatory="APO", columns += [ fits.Column(name="mjd", array=[59804], format="J"), ] + flux_col = "model_flux" if astra else "flux" + columns += [ - fits.Column(name="flux", array=flux, format="8575E", dim="(8575)"), + fits.Column(name=flux_col, array=flux, format="8575E", dim="(8575)"), fits.Column(name="ivar", array=ivar, format="8575E", dim="(8575)"), fits.Column(name="pixel_flags", array=pixel_flags, @@ -84,7 +87,11 @@ def generate_apogee_hdu(observatory="APO", return fits.BinTableHDU.from_columns(columns, header=header) -def generate_boss_hdu(observatory="APO", with_wl=True, datasum="0", nvisits=1): +def generate_boss_hdu(observatory="APO", + with_wl=True, + datasum="0", + nvisits=1, + astra=False): wl = (10**(3.5523 + 1e-4 * np.arange(4648))).reshape((1, -1)) flux = np.array([np.zeros_like(wl)] * nvisits) ivar = np.array([np.zeros_like(wl)] * nvisits) @@ -92,15 +99,14 @@ def generate_boss_hdu(observatory="APO", with_wl=True, datasum="0", nvisits=1): continuum = np.array([np.zeros_like(wl)] * nvisits) nmf_rectified_model_flux = np.array([np.zeros_like(wl)] * nvisits) columns = [ - fits.Column(name="spectrum_pk_id", array=[0], format="K"), - fits.Column(name="release", array=["sdss5"], format="5A"), - fits.Column(name="filetype", array=["specFull"], format="7A"), + fits.Column(name="spectrum_pk_id", array=[0] * nvisits, format="K"), + fits.Column(name="release", array=["sdss5"] * nvisits, format="5A"), fits.Column(name="v_astra", array=["0.5.0"], format="5A"), fits.Column(name="healpix", array=[34], format="J"), fits.Column(name="sdss_id", array=[42], format="K"), fits.Column(name="run2d", array=["6_1_2"], format="6A"), - fits.Column(name="telescope", array=["apo25m"], format="6A"), - fits.Column(name="snr", array=[50], format="E"), + fits.Column(name="telescope", array=["apo25m"] * nvisits, format="6A"), + fits.Column(name="snr", array=[50] * nvisits, format="E"), ] if with_wl: @@ -117,8 +123,9 @@ def generate_boss_hdu(observatory="APO", with_wl=True, datasum="0", nvisits=1): columns += [ fits.Column(name="mjd", array=[59804], format="J"), ] + flux_col = "model_flux" if astra else "flux" columns += [ - fits.Column(name="flux", array=flux, format="4648E", dim="(4648)"), + fits.Column(name=flux_col, array=flux, format="4648E", dim="(4648)"), fits.Column(name="ivar", array=ivar, format="4648E", dim="(4648)"), fits.Column(name="pixel_flags", array=pixel_flags, @@ -464,7 +471,50 @@ def test_mwm_1d_nohdu(file_obj, hdu, with_wl, hduflags, nvisits): assert data.flux.value.shape[-1] == length if nvisits > 1: assert data.flux.value.shape[0] == nvisits + if with_wl: + assert data.meta["datatype"].lower() == "mwmstar" + else: + assert data.meta["datatype"].lower() == "mwmvisit" + assert data.spectral_axis.unit == Angstrom + assert data.flux.unit == Unit("1e-17 erg / (s cm2 Angstrom)") + os.remove(tmpfile) + +@pytest.mark.parametrize( + "file_obj, hdu, with_wl, hduflags, nvisits", + [ + ("mwm-temp", None, False, [0, 0, 1, 0], 1), # visit + ("mwm-temp", None, False, [0, 1, 1, 0], 3), # multi-ext visits + ("mwm-temp", None, True, [0, 0, 1, 0], 1), # star + ("mwm-temp", None, True, [0, 1, 1, 0], 1), + ], +) +def test_astra_nohdu(file_obj, hdu, with_wl, hduflags, nvisits): + """Test astra Spectrum1D loader when HDU isn't specified""" + tmpfile = str(file_obj) + ".fits" + mwm_HDUList(hduflags, with_wl, nvisits=nvisits, + astra=True).writeto(tmpfile, overwrite=True) + + with pytest.warns(AstropyUserWarning): + data = Spectrum1D.read(tmpfile, hdu=hdu) + assert isinstance(data, Spectrum1D) + assert isinstance(data.meta["header"], fits.Header) + + if data.meta["instrument"].lower() == "apogee": + length = 8575 + elif data.meta["instrument"].lower() == "boss": + length = 4648 + else: + raise ValueError( + "INSTRMNT tag in test HDU header is not set properly.") + assert len(data.spectral_axis.value) == length + assert data.flux.value.shape[-1] == length + if nvisits > 1: + assert data.flux.value.shape[0] == nvisits + if with_wl: + assert data.meta["datatype"].lower() == "astrastar" + else: + assert data.meta["datatype"].lower() == "astravisit" assert data.spectral_axis.unit == Angstrom assert data.flux.unit == Unit("1e-17 erg / (s cm2 Angstrom)") os.remove(tmpfile) @@ -500,6 +550,49 @@ def test_mwm_1d(file_obj, hdu, with_wl, hduflags, nvisits): assert data.flux.value.shape[-1] == length if nvisits > 1: assert data.flux.value.shape[0] == nvisits + if with_wl: + assert data.meta["datatype"].lower() == "mwmstar" + else: + assert data.meta["datatype"].lower() == "mwmvisit" + assert data.spectral_axis.unit == Angstrom + assert data.flux.unit == Unit("1e-17 erg / (s cm2 Angstrom)") + os.remove(tmpfile) + + +@pytest.mark.parametrize( + "file_obj, hdu, with_wl, hduflags, nvisits", + [ + ("astra-temp", 3, False, [0, 0, 1, 0], 1), + ("astra-temp", 3, False, [0, 1, 1, 0], 5), + ("astra-temp", 3, True, [0, 0, 1, 0], 1), + ("astra-temp", 2, True, [0, 1, 1, 0], 1), + ], +) +def test_astra_1d(file_obj, hdu, with_wl, hduflags, nvisits): + """Test astra Spectrum1D loader""" + tmpfile = str(file_obj) + ".fits" + mwm_HDUList(hduflags, with_wl, nvisits=nvisits, + astra=True).writeto(tmpfile, overwrite=True) + + data = Spectrum1D.read(tmpfile, hdu=hdu) + assert isinstance(data, Spectrum1D) + assert isinstance(data.meta["header"], fits.Header) + if data.meta["instrument"].lower() == "apogee": + length = 8575 + elif data.meta["instrument"].lower() == "boss": + length = 4648 + else: + raise ValueError( + "INSTRMNT tag in test HDU header is not set properly.") + assert len(data.spectral_axis.value) == length + assert data.flux.value.shape[-1] == length + if nvisits > 1: + assert data.flux.value.shape[0] == nvisits + + if with_wl: + assert data.meta["datatype"].lower() == "astrastar" + else: + assert data.meta["datatype"].lower() == "astravisit" assert data.spectral_axis.unit == Angstrom assert data.flux.unit == Unit("1e-17 erg / (s cm2 Angstrom)") @@ -539,9 +632,54 @@ def test_mwm_list(file_obj, with_wl, hduflags): raise ValueError( "INSTRMNT tag in test HDU header is not set properly.") if with_wl: - assert data[i].meta['datatype'].lower() == 'mwmstar' + assert data[i].meta["datatype"].lower() == "mwmstar" else: - assert data[i].meta['datatype'].lower() == 'mwmvisit' + assert data[i].meta["datatype"].lower() == "mwmvisit" + assert len(data[i].spectral_axis.value) == length + assert data[i].flux.value.shape[-1] == length + if nvisits > 1: + assert data[i].flux.value.shape[0] == nvisits + assert data[i].spectral_axis.unit == Angstrom + assert data[i].flux.unit == Unit("1e-17 erg / (s cm2 Angstrom)") + os.remove(tmpfile) + + +@pytest.mark.parametrize( + "file_obj, with_wl, hduflags", + [ + ("astra-temp", False, [0, 0, 1, 1]), + ("astra-temp", False, [0, 1, 1, 0]), + ("astra-temp", False, [1, 1, 0, 0]), + ("astra-temp", False, [1, 1, 1, 1]), + ("astra-temp", True, [0, 0, 1, 1]), + ("astra-temp", True, [0, 1, 1, 0]), + ("astra-temp", True, [1, 1, 0, 0]), + ("astra-temp", True, [1, 1, 1, 1]), + ], +) +def test_astra_list(file_obj, with_wl, hduflags): + """Test astra SpectrumList loader""" + tmpfile = str(file_obj) + ".fits" + nvisits = 1 if with_wl else 3 + mwm_HDUList(hduflags, with_wl, nvisits=nvisits, + astra=True).writeto(tmpfile, overwrite=True) + + data = SpectrumList.read(tmpfile) + assert isinstance(data, SpectrumList) + for i in range(len(data)): + assert isinstance(data[i], Spectrum1D) + assert isinstance(data[i].meta["header"], fits.Header) + if data[i].meta["instrument"].lower() == "apogee": + length = 8575 + elif data[i].meta["instrument"].lower() == "boss": + length = 4648 + else: + raise ValueError( + "INSTRMNT tag in test HDU header is not set properly.") + if with_wl: + assert data[i].meta["datatype"].lower() == "astrastar" + else: + assert data[i].meta["datatype"].lower() == "astravisit" assert len(data[i].spectral_axis.value) == length assert data[i].flux.value.shape[-1] == length if nvisits > 1: @@ -586,6 +724,24 @@ def test_mwm_1d_fail(file_obj, with_wl): os.remove(tmpfile) +@pytest.mark.parametrize( + "file_obj, with_wl", + [ + ("astra-temp", False), + ("astra-temp", True), + ], +) +def test_astra_1d_fail(file_obj, with_wl): + """Test astra Spectrum1D loader fail on empty""" + tmpfile = str(file_obj) + ".fits" + mwm_HDUList([0, 0, 0, 0], with_wl, astra=True).writeto(tmpfile, + overwrite=True) + + with pytest.raises(ValueError): + Spectrum1D.read(tmpfile) + os.remove(tmpfile) + + @pytest.mark.parametrize( "file_obj, with_wl", [ @@ -603,6 +759,24 @@ def test_mwm_list_fail(file_obj, with_wl): os.remove(tmpfile) +@pytest.mark.parametrize( + "file_obj, with_wl", + [ + ("astra-temp", False), + ("astra-temp", True), + ], +) +def test_astra_list_fail(file_obj, with_wl): + """Test astra SpectrumList loader fail on empty""" + tmpfile = str(file_obj) + ".fits" + mwm_HDUList([0, 0, 0, 0], with_wl, astra=True).writeto(tmpfile, + overwrite=True) + + with pytest.raises(ValueError): + SpectrumList.read(tmpfile) + os.remove(tmpfile) + + @pytest.mark.parametrize( "file_obj,n_spectra", [