From 1d060673ae01bdc683c54c7e54ac180f685e0a72 Mon Sep 17 00:00:00 2001 From: Dan Avner Date: Tue, 17 Oct 2023 15:49:55 -0700 Subject: [PATCH 1/2] Extend Gemini to include getting file content and logging out of the Archive. --- astroquery/gemini/core.py | 104 +++++++++++++++++++++++-- astroquery/gemini/tests/test_gemini.py | 43 +++++++++- astroquery/gemini/tests/test_remote.py | 16 ++++ docs/gemini/gemini.rst | 21 +++-- 4 files changed, 172 insertions(+), 12 deletions(-) diff --git a/astroquery/gemini/core.py b/astroquery/gemini/core.py index dbe7c8d38a..af30098cd3 100644 --- a/astroquery/gemini/core.py +++ b/astroquery/gemini/core.py @@ -1,20 +1,21 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst """ -Search functionality for the Gemini archive of observations. +================================================== +Gemini Observatory Archive (GOA) Astroquery Module +================================================== -For questions, contact ooberdorf@gemini.edu +Query public and proprietary data from GOA. """ - import os - from datetime import date from astroquery import log +import astropy from astropy import units from astropy.table import Table, MaskedColumn - -from astroquery.gemini.urlhelper import URLHelper import numpy as np +from .urlhelper import URLHelper from ..query import QueryWithLogin from ..utils.class_or_instance import class_or_instance from . import conf @@ -433,6 +434,97 @@ def get_file(self, filename, *, download_dir='.', timeout=None): local_filepath = os.path.join(download_dir, filename) self._download_file(url=url, local_filepath=local_filepath, timeout=timeout) + def _download_file_content(self, url, timeout=None, auth=None, method="GET", **kwargs): + """Download content from a URL and return it. Resembles + `_download_file` but returns the content instead of saving it to a + local file. + + Parameters + ---------- + url : str + The URL from where to download the file. + timeout : int, optional + Time in seconds to wait for the server response, by default + `None`. + auth : dict[str, Any], optional + Authentication details, by default `None`. + method : str, optional + The HTTP method to use, by default "GET". + + Returns + ------- + bytes + The downloaded content. + """ + + response = self._session.request(method, url, timeout=timeout, auth=auth, **kwargs) + response.raise_for_status() + + if 'content-length' in response.headers: + length = int(response.headers['content-length']) + if length == 0: + log.warn(f'URL {url} has length=0') + + blocksize = astropy.utils.data.conf.download_block_size + content = b"" + + for block in response.iter_content(blocksize): + content += block + + response.close() + + return content + + def logout(self): + """Logout from the GOA service by deleting the specific session cookie + and updating the authentication state. + """ + # Delete specific cookie. + cookie_name = "gemini_archive_session" + if cookie_name in self._session.cookies: + del self._session.cookies[cookie_name] + + # Update authentication state. + self._authenticated = False + + def get_file_content(self, filename, timeout=None, auth=None, method="GET", **kwargs): + """Wrapper around `_download_file_content`. + + Parameters + ---------- + filename : str + Name of the file to download content. + timeout : int, optional + Time in seconds to wait for the server response, by default + `None`. + auth : dict[str, Any], optional + Authentication details, by default `None`. + method : str, optional + The HTTP method to use, by default "GET". + + Returns + ------- + bytes + The downloaded content. + """ + url = self.get_file_url(filename) + return self._download_file_content(url, timeout=timeout, auth=auth, method=method, **kwargs) + + def get_file_url(self, filename): + """Generate the file URL based on the filename. + + Parameters + ---------- + filename : str + The name of the file. + + Returns + ------- + str + The URL where the file can be downloaded. + """ + return f"https://archive.gemini.edu/file/{filename}" + def _gemini_json_to_table(json): """ diff --git a/astroquery/gemini/tests/test_gemini.py b/astroquery/gemini/tests/test_gemini.py index a3a36a7f9b..1c5fe57ef7 100644 --- a/astroquery/gemini/tests/test_gemini.py +++ b/astroquery/gemini/tests/test_gemini.py @@ -24,10 +24,21 @@ class MockResponse: def __init__(self, text): self.text = text + self.headers = {'content-length': str(len(text))} + self.status_code = 200 def json(self): return json.loads(self.text) + def raise_for_status(self): + pass + + def iter_content(self, blocksize): + yield self.text + + def close(self): + pass + @pytest.fixture def patch_get(request): @@ -35,7 +46,15 @@ def patch_get(request): mp = request.getfixturevalue("monkeypatch") mp.setattr(requests.Session, 'request', get_mockreturn) - return mp + + +@pytest.fixture +def patch_content(monkeypatch): + """Mock requests with encoded content.""" + def mock_request(*args, **kwargs): + return MockResponse(b"mock_content") + + monkeypatch.setattr(requests.Session, 'request', mock_request) # to inspect behavior, updated when the mock get call is made @@ -171,3 +190,25 @@ def test_url_helper_eng_fail(test_arg): urlsplit = url.split('/') assert (('notengineering' in urlsplit) == should_have_noteng) assert (('NotFail' in urlsplit) == should_have_notfail) + + +def test_logout(): + """Test logout functionality.""" + gemini.Observations._session.cookies = {"gemini_archive_session": "some_value"} + gemini.Observations._authenticated = True + gemini.Observations.logout() + assert "gemini_archive_session" not in gemini.Observations._session.cookies + assert gemini.Observations._authenticated is False + + +def test_get_file_content(patch_content): + """Test wrapper around _download_file_content.""" + content = gemini.Observations.get_file_content("filename", timeout=5) + assert content == b"mock_content" + + +def test_get_file_url(): + """Test generating file URL based on filename.""" + url = gemini.Observations.get_file_url("filename") + assert url == "https://archive.gemini.edu/file/filename" + diff --git a/astroquery/gemini/tests/test_remote.py b/astroquery/gemini/tests/test_remote.py index e5c5c9bd12..a1a9801272 100644 --- a/astroquery/gemini/tests/test_remote.py +++ b/astroquery/gemini/tests/test_remote.py @@ -8,6 +8,7 @@ https://astroquery.readthedocs.io/en/latest/testing.html """ import pytest +import requests import os import shutil @@ -23,6 +24,10 @@ """ Coordinates to use for testing """ coords = SkyCoord(210.80242917, 54.34875, unit="deg") +# Filename and url +filename = "S20231016S0018.fits.bz2" # Small file +file_url = f"https://archive.gemini.edu/file{filename}" + @pytest.mark.remote_data class TestGemini: @@ -78,3 +83,14 @@ def test_get_file(self): os.unlink(filepath) if os.path.exists(tempdir): shutil.rmtree(tempdir) + + def test_get_file_content(self): + """Test the `get_file_content` function.""" + content = gemini.Observations.get_file_content(filename) + assert isinstance(content, bytes) + assert len(content) > 0 + + def test_get_file_content_with_timeout(self): + """Test `get_file_content` with a timeout.""" + with pytest.raises(requests.exceptions.Timeout): + gemini.Observations.get_file_content(filename, timeout=0.001) diff --git a/docs/gemini/gemini.rst b/docs/gemini/gemini.rst index b1616b42f2..ac83ac5f25 100644 --- a/docs/gemini/gemini.rst +++ b/docs/gemini/gemini.rst @@ -132,19 +132,21 @@ the *NotFail* or *notengineering* terms respectively. Authenticated Sessions ---------------------- -The Gemini module allows for authenticated sessions using your GOA account. This is the same account you login -with on the GOA homepage at ``__. The `astroquery.gemini.ObservationsClass.login` -method returns `True` if successful. +The Gemini module allows for authenticated sessions using your GOA account. This is the same account you +login with on the GOA homepage at ``__. The +`astroquery.gemini.ObservationsClass.login` method returns `True` if successful. To logout, use the +`astroquery.gemini.ObservationsClass.logout` method to remove the Gemini Observatory Archive session cookie. .. doctest-skip:: >>> from astroquery.gemini import Observations >>> Observations.login(username, password) >>> # do something with your elevated access + >>> Observations.logout() -File Downloading ----------------- +File Downloading and File Content Getting +----------------------------------------- As a convenience, you can request file downloads directly from the Gemini module. This constructs the appropriate URL and fetches the file. It will use any authenticated session you may have, so it will retrieve any @@ -156,6 +158,15 @@ proprietary data you may be permissioned for. >>> Observations.get_file("GS2020AQ319-10.fits", download_dir="/tmp") # doctest: +IGNORE_OUTPUT +To get the file content without writing to disk, you can use the method +`astroquery.gemini.ObservationsClass.get_file_content`. This constructs the appropriate url and fetches the +file contents. This will use any authenticated session you have for proprietary data. + +.. doctest-remote-data:: + >>> from astroquery.gemini import Observations + >>> Observations.get_file_content("GS2020AQ319-10.fits") # doctest: +IGNORE_OUTPUT + + Reference/API ============= From 348ef141c6e91390ed7adfa24c3902a50227dcc5 Mon Sep 17 00:00:00 2001 From: Dan Avner Date: Tue, 17 Oct 2023 16:00:32 -0700 Subject: [PATCH 2/2] Address PEP 8 blank line. --- astroquery/gemini/tests/test_gemini.py | 1 - 1 file changed, 1 deletion(-) diff --git a/astroquery/gemini/tests/test_gemini.py b/astroquery/gemini/tests/test_gemini.py index 1c5fe57ef7..4669eb7878 100644 --- a/astroquery/gemini/tests/test_gemini.py +++ b/astroquery/gemini/tests/test_gemini.py @@ -211,4 +211,3 @@ def test_get_file_url(): """Test generating file URL based on filename.""" url = gemini.Observations.get_file_url("filename") assert url == "https://archive.gemini.edu/file/filename" -