From 2f990b4e44e26cb2db41f5f575851ad043c7394d Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Thu, 27 Jun 2024 15:46:31 -0400 Subject: [PATCH 1/3] Update MastMissions documentation --- docs/mast/mast_missions.rst | 135 +++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 64 deletions(-) diff --git a/docs/mast/mast_missions.rst b/docs/mast/mast_missions.rst index f3dedb685a..04eb93cda8 100644 --- a/docs/mast/mast_missions.rst +++ b/docs/mast/mast_missions.rst @@ -7,17 +7,17 @@ Mission-Specific Search Queries =============================== These queries allow for searches based on mission-specific metadata for a given -data collection. Currently it provides access to a broad set of Hubble Space -Telescope (HST) metadata, including header keywords, proposal information, and -observational parameters. The available metadata includes all information that -was previously available in the original HST web search form, and are present in -the current `Mission Search interface `__. +data collection. Currently, it provides access to a broad set of Hubble Space +Telescope (HST) and James Webb Space Telescope (JWST) metadata, including header keywords, +proposal information, and observational parameters. -**Note:** this API interface does not yet support data product download, only -metadata earch access. +**Note:** This API interface does not yet support data product downloads, only +metadata search access. -An object of MastMissions class is instantiated with a default mission of 'hst' and -default service set to 'search'. +An object of the ``MastMissions`` class is instantiated with a default mission of ``'hst'`` and +default service set to ``'search'``. The searchable metadata for Hubble encompasses all information that +was previously accessible through the original HST web search form and is now available in +the current `MAST HST Search Form `__. .. doctest-remote-data:: @@ -28,11 +28,21 @@ default service set to 'search'. >>> missions.service 'search' -The missions object can be used to search metadata using by sky position, or other criteria. -The keyword arguments can be used to specify output characteristics like selec_cols and -sort_by and conditions that filter on values like proposal id, pi last name etc. -The available column names for a mission are returned by the -`~astroquery.mast.MastMissionsClass.get_column_list` function. +To search for JWST metadata, a ``MastMissions`` object is instantiated with a value of ``'jwst'`` for ``mission``. +The searchable metadata for Webb encompasses all information that is available in +the current `MAST JWST Search Form `__. + +.. doctest-remote-data:: + + >>> from astroquery.mast import MastMissions + >>> missions = MastMissions(mission='jwst') + >>> missions.mission + 'jwst' + +The ``missions`` object can be used to search metadata by object name, sky position, or other criteria. +When writing queries, keyword arguments can be used to specify output characteristics and filter on +values like instrument, exposure type, and principal investigator. The available column names for a +mission are returned by the `~astroquery.mast.MastMissionsClass.get_column_list` function. .. doctest-remote-data:: @@ -40,12 +50,12 @@ The available column names for a mission are returned by the >>> missions = MastMissions(mission='hst') >>> columns = missions.get_column_list() -For positional searches, the columns "ang_sep", "sci_data_set_name", "search_key" and "search_position" -will always be included, in addition to any columns specified using "select_cols". For non-positional -searches, "search_key" and "sci_data_set_name" will always be included, in addition to any columns -specified using "select_cols". +Metadata queries can be performed on a particular region in the sky. Passing in a set of coordinates to the +`~astroquery.mast.MastMissionsClass.query_region` function returns datasets that fall within a +certain radius value of that point. This type of search is also known as a cone search. -For a non positional search, ``select_cols`` would always include ``'search_key'`` and ``'sci_data_set_name'``. +The ``select_cols`` keyword argument specifies a list of columns to be included in the response. +The ``sort_by`` keyword argument specifies a column (or columns) to sort the results by. .. doctest-remote-data:: @@ -53,64 +63,61 @@ For a non positional search, ``select_cols`` would always include ``'search_key' >>> from astropy.coordinates import SkyCoord >>> missions = MastMissions(mission='hst') >>> regionCoords = SkyCoord(210.80227, 54.34895, unit=('deg', 'deg')) - >>> results = missions.query_region(regionCoords, radius=3, sci_pep_id=12556, + >>> results = missions.query_region(regionCoords, + ... radius=3, + ... sci_pep_id=12556, ... select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"], ... sort_by=['sci_targname']) >>> results[:5] # doctest: +IGNORE_OUTPUT - sci_status sci_targname sci_data_set_name ang_sep sci_pep_id search_pos sci_pi_last_name search_key - str6 str16 str9 str20 int64 str18 str6 str27 - ---------- ---------------- ----------------- -------------------- ---------- ------------------ ---------------- --------------------------- - PUBLIC NUCLEUS+HODGE602 OBQU010H0 0.017460048037303017 12556 210.80227 54.34895 GORDON 210.80227 54.34895OBQU010H0 - PUBLIC NUCLEUS+HODGE602 OBQU01050 0.017460048037303017 12556 210.80227 54.34895 GORDON 210.80227 54.34895OBQU01050 - PUBLIC NUCLEUS+HODGE602 OBQU01030 0.022143836477276503 12556 210.80227 54.34895 GORDON 210.80227 54.34895OBQU01030 - PUBLIC NUCLEUS+HODGE602 OBQU010F0 0.022143836477276503 12556 210.80227 54.34895 GORDON 210.80227 54.34895OBQU010F0 - PUBLIC NUCLEUS+HODGE602 OBQU010J0 0.04381046755938432 12556 210.80227 54.34895 GORDON 210.80227 54.34895OBQU010J0 - -for paging through the results, offset and limit can be used to specify the starting record and the number -of returned records. the default values for offset and limit is 0 and 5000 respectively. + search_pos sci_data_set_name sci_targname sci_start_time sci_stop_time ang_sep sci_status + ------------------ ----------------- ---------------- -------------------------- -------------------------- -------------------- ---------- + 210.80227 54.34895 OBQU01050 NUCLEUS+HODGE602 2012-05-24T07:51:40.553000 2012-05-24T07:54:46.553000 0.017460048037303017 PUBLIC + 210.80227 54.34895 OBQU010H0 NUCLEUS+HODGE602 2012-05-24T09:17:38.570000 2012-05-24T09:20:44.570000 0.017460048037303017 PUBLIC + 210.80227 54.34895 OBQU01030 NUCLEUS+HODGE602 2012-05-24T07:43:20.553000 2012-05-24T07:46:26.553000 0.022143836477276503 PUBLIC + 210.80227 54.34895 OBQU010F0 NUCLEUS+HODGE602 2012-05-24T09:09:18.570000 2012-05-24T09:12:24.570000 0.022143836477276503 PUBLIC + 210.80227 54.34895 OBQU01070 NUCLEUS+HODGE602 2012-05-24T08:00:00.553000 2012-05-24T08:03:06.553000 0.04381046755938432 PUBLIC -.. doctest-remote-data:: +You may notice that the above query returned more columns than were specified in the ``select_cols`` +argument. For each mission, certain columns are automatically returned. - >>> from astroquery.mast import MastMissions - >>> from astropy.coordinates import SkyCoord - >>> missions = MastMissions() - >>> results = missions.query_criteria(sci_start_time=">=2021-01-01 00:00:00", - ... select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status", "sci_pep_id"], - ... sort_by=['sci_pep_id'], limit=1000, offset=1000) # doctest: +IGNORE_WARNINGS - ... # MaxResultsWarning('Maximum results returned, may not include all sources within radius.') - >>> len(results) - 1000 +* *HST*: For positional searches, the columns ``ang_sep``, ``sci_data_set_name``, and ``search_pos`` + are always included in the query results. For non-positional searches, ``sci_data_set_name`` is always + present. -Metadata queries can also be performed using object names with the -~astroquery.mast.MastMissionsClass.query_object function. +* *JWST*: For every query, the ``ArchiveFileID`` column is always returned. + +Searches can also be run on target names with the `~astroquery.mast.MastMissionsClass.query_object` +function. .. doctest-remote-data:: - >>> results = missions.query_object('M101', radius=3, select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"], + >>> results = missions.query_object('M101', + ... radius=3, + ... select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"], ... sort_by=['sci_targname']) >>> results[:5] # doctest: +IGNORE_OUTPUT
- ang_sep search_pos sci_status search_key sci_stop_time sci_targname sci_start_time sci_data_set_name - str20 str18 str6 str27 str26 str16 str26 str9 - ------------------ ------------------ ---------- --------------------------- -------------------------- ------------ -------------------------- ----------------- - 2.751140575012458 210.80227 54.34895 PUBLIC 210.80227 54.34895LDJI01010 2019-02-19T05:52:40.020000 +164.6+9.9 2019-02-19T00:49:58.010000 LDJI01010 - 0.8000626246647815 210.80227 54.34895 PUBLIC 210.80227 54.34895J8OB02011 2003-08-27T08:27:34.513000 ANY 2003-08-27T07:44:47.417000 J8OB02011 - 1.1261718338567348 210.80227 54.34895 PUBLIC 210.80227 54.34895J8D711J1Q 2003-01-17T00:50:22.250000 ANY 2003-01-17T00:42:06.993000 J8D711J1Q - 1.1454431087675097 210.80227 54.34895 PUBLIC 210.80227 54.34895JD6V01012 2017-06-15T18:33:25.983000 ANY 2017-06-15T18:10:12.037000 JD6V01012 - 1.1457795862361977 210.80227 54.34895 PUBLIC 210.80227 54.34895JD6V01013 2017-06-15T20:08:44.063000 ANY 2017-06-15T19:45:30.023000 JD6V01013 - -Metadata queries can also be performed using non-positional parameters with the -`~astroquery.mast.MastMissionsClass.query_criteria` function. + search_pos sci_data_set_name sci_targname sci_start_time sci_stop_time ang_sep sci_status + ------------------ ----------------- ------------ -------------------------- -------------------------- ------------------ ---------- + 210.80243 54.34875 LDJI01010 +164.6+9.9 2019-02-19T00:49:58.010000 2019-02-19T05:52:40.020000 2.7469653000840397 PUBLIC + 210.80243 54.34875 J8OB02011 ANY 2003-08-27T07:44:47.417000 2003-08-27T08:27:34.513000 0.8111299061221189 PUBLIC + 210.80243 54.34875 J8D711J1Q ANY 2003-01-17T00:42:06.993000 2003-01-17T00:50:22.250000 1.1297984178946574 PUBLIC + 210.80243 54.34875 JD6V01012 ANY 2017-06-15T18:10:12.037000 2017-06-15T18:33:25.983000 1.1541053362381077 PUBLIC + 210.80243 54.34875 JD6V01013 ANY 2017-06-15T19:45:30.023000 2017-06-15T20:08:44.063000 1.15442580192948 PUBLIC + +For non-positional metadata queries, use the `~astroquery.mast.MastMissionsClass.query_criteria` +function. For paging through results, the ``offset`` and ``limit`` keyword arguments can be used +to specify the starting record and the number of returned records. The default values for ``offset`` +and ``limit`` are 0 and 5000, respectively. .. doctest-remote-data:: - >>> results = missions.query_criteria(sci_data_set_name="Z06G0101T", sci_pep_id="1455", - ... select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status"], - ... sort_by=['sci_targname']) - >>> results[:5] # doctest: +IGNORE_OUTPUT -
- search_key sci_stop_time sci_data_set_name sci_start_time sci_targname sci_status - str9 str26 str9 str26 str19 str6 - ---------- -------------------------- ----------------- -------------------------- ------------ ---------- - Z06G0101T 1990-05-13T11:02:34.567000 Z06G0101T 1990-05-13T10:38:09.193000 -- PUBLIC \ No newline at end of file + >>> results = missions.query_criteria(sci_start_time=">=2021-01-01 00:00:00", + ... select_cols=["sci_stop_time", "sci_targname", "sci_start_time", "sci_status", "sci_pep_id"], + ... sort_by=['sci_pep_id'], + ... limit=1000, + ... offset=1000) # doctest: +IGNORE_WARNINGS + ... # MaxResultsWarning('Maximum results returned, may not include all sources within radius.') + >>> len(results) + 1000 From faba68f7a10bec4764ed936aa7ac8a00dd028a1a Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Fri, 28 Jun 2024 14:32:37 -0400 Subject: [PATCH 2/3] Remote tests, send in requests in arcseconds --- astroquery/mast/missions.py | 8 +-- astroquery/mast/tests/test_mast_remote.py | 87 ++++++++++++++++++++++- 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/astroquery/mast/missions.py b/astroquery/mast/missions.py index 38738f461a..18df28ae4d 100644 --- a/astroquery/mast/missions.py +++ b/astroquery/mast/missions.py @@ -112,8 +112,8 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs # basic params params = {'target': [f"{coordinates.ra.deg} {coordinates.dec.deg}"], - 'radius': radius.arcmin, - 'radius_units': 'arcminutes', + 'radius': radius.arcsec, + 'radius_units': 'arcseconds', 'limit': limit, 'offset': offset} @@ -180,8 +180,8 @@ def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u. params = {"limit": self.limit, "offset": offset, 'select_cols': select_cols} if coordinates: params["target"] = [f"{coordinates.ra.deg} {coordinates.dec.deg}"] - params["radius"] = radius.arcmin - params["radius_units"] = 'arcminutes' + params["radius"] = radius.arcsec + params["radius_units"] = 'arcseconds' if not self._service_api_connection.check_catalogs_criteria_params(criteria): raise InvalidQueryError("At least one non-positional criterion must be supplied.") diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index e1ddfea34b..531115e7e6 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -12,7 +12,7 @@ from astropy.io import fits import astropy.units as u -from astroquery.mast import Observations, utils, Mast, Catalogs, Hapcut, Tesscut, Zcut +from astroquery.mast import Observations, utils, Mast, Catalogs, Hapcut, Tesscut, Zcut, MastMissions from ..utils import ResolverError from ...exceptions import (InputWarning, InvalidQueryError, MaxResultsWarning, @@ -51,6 +51,91 @@ def test_resolve_object(self): ticobj_loc = utils.resolve_object("TIC 141914082") assert round(ticobj_loc.separation(SkyCoord("94.6175354 -72.04484622", unit='deg')).value, 4) == 0 + ########################### + # MissionSearchClass Test # + ########################### + + def test_missions_get_column_list(self): + columns = MastMissions().get_column_list() + assert len(columns) > 1 + assert isinstance(columns, Table) + assert list(columns.columns.keys()) == ['name', 'data_type', 'description'] + + def test_missions_query_region_async(self): + coords = SkyCoord(83.6287, 22.0147, unit="deg") + response = MastMissions.query_region_async(coords, radius=1) + assert isinstance(response, Response) + assert response.status_code == 200 + + def test_missions_query_region(self): + select_cols = ['sci_targname', 'sci_instrume'] + result = MastMissions.query_region("245.89675 -26.52575", + radius=0.1, + sci_instrume="WFC3, ACS", + select_cols=select_cols + ) + assert isinstance(result, Table) + assert len(result) > 0 + assert (result['ang_sep'].data.data.astype('float') < 0.1).all() + ins_strip = np.char.strip(result['sci_instrume'].data) + assert ((ins_strip == 'WFC3') | (ins_strip == 'ACS')).all() + assert all(c in list(result.columns.keys()) for c in select_cols) + + def test_missions_query_object_async(self): + response = MastMissions.query_object_async("M4", radius=0.1) + assert isinstance(response, Response) + assert response.status_code == 200 + + def test_missions_query_object(self): + result = MastMissions.query_object("NGC6121", + radius=6*u.arcsec, + sci_pi_last_name='*LE*', + sci_spec_1234='!F395N' + ) + assert isinstance(result, Table) + assert len(result) > 0 + assert "NGC6121" in result["sci_targname"] + assert (result['ang_sep'].data.data.astype('float') < 0.1).all() + assert (result['sci_pi_last_name'] == 'LEE').all() + assert 'F395N' not in result['sci_spec_1234'] + + def test_missions_query_criteria_async(self): + response = MastMissions.query_criteria_async(sci_pep_id=12557, + sci_obs_type='SPECTRUM', + sci_aec='S') + assert isinstance(response, Response) + assert response.status_code == 200 + + def test_missions_query_criteria(self): + # Non-positional search + with pytest.warns(MaxResultsWarning): + result = MastMissions.query_criteria(sci_pep_id=12557, + sci_obs_type='SPECTRUM', + sci_aec='S', + limit=3, + select_cols=['sci_pep_id', 'sci_obs_type', 'sci_aec']) + assert isinstance(result, Table) + assert len(result) == 3 + assert (result['sci_pep_id'] == 12557).all() + assert (result['sci_obs_type'] == 'SPECTRUM').all() + assert (result['sci_aec'] == 'S').all() + + # Positional criteria search + result = MastMissions.query_criteria(objectname='NGC6121', + radius=0.1, + sci_start_time='<2012', + sci_actual_duration='0..200' + ) + assert len(result) == 3 + assert (result['ang_sep'].data.data.astype('float') < 0.1).all() + assert (result['sci_start_time'] < '2012').all() + assert ((result['sci_actual_duration'] >= 0) & (result['sci_actual_duration'] <= 200)).all() + + # Raise error if a non-positional criterion is not supplied + with pytest.raises(InvalidQueryError): + MastMissions.query_criteria(coordinates="245.89675 -26.52575", + radius=1) + ################### # MastClass tests # ################### From 2a5156bffa8cb556608747762ab2c2d90da9cb14 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Fri, 28 Jun 2024 15:37:23 -0400 Subject: [PATCH 3/3] Modify service mock return to reflect arcseconds --- astroquery/mast/tests/test_mast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index dd048d43a2..d1fefec201 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -119,7 +119,7 @@ def service_mockreturn(self, method="POST", url=None, data=None, timeout=10, use filename = data_path(DATA_FILES['z_survey']) else: filename = data_path(DATA_FILES['z_cutout_fit']) - elif use_json and data['radius'] == 5: + elif use_json and data['radius'] == 300: filename = data_path(DATA_FILES["mission_incorrect_results"]) elif use_json: filename = data_path(DATA_FILES["mission_search_results"])