From c2092242ad2b3aa01cf0c2e192cc175a903cf2fc Mon Sep 17 00:00:00 2001 From: Daralynn Rhode Date: Mon, 31 Mar 2025 14:39:59 -0600 Subject: [PATCH 1/8] adding ingestion to cli.py --- imap_data_access/cli.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/imap_data_access/cli.py b/imap_data_access/cli.py index 34ddd46..6d41525 100644 --- a/imap_data_access/cli.py +++ b/imap_data_access/cli.py @@ -57,6 +57,7 @@ def _print_query_results_table(query_results: list[dict]): "Data Level", "Descriptor", "Start Date", + "Ingestion Date", "Repointing", "Version", "Filename", @@ -101,6 +102,7 @@ def _print_query_results_table(query_results: list[dict]): str(item.get("data_level", "")), str(item.get("descriptor", "")), str(item.get("start_date", "")), + str(item.get("ingestion_date", "")), str(item.get("repointing", "")) or "", str(item.get("version", "")), os.path.basename(item.get("file_path", "")), @@ -126,6 +128,8 @@ def _query_parser(args: argparse.Namespace): "descriptor", "start_date", "end_date", + "ingestion_start_date", + "ingestion_end_date", "repointing", "version", "extension", @@ -334,6 +338,18 @@ def main(): # noqa: PLR0915 required=False, help="End date for a range of file timestamps in YYYYMMDD format", ) + query_parser.add_argument( + "--ingestion-start-date", + type=str, + required=False, + help="Ingestion start date by for files in YYYYMMDD format", + ) + query_parser.add_argument( + "--ingestion-end-date", + type=str, + required=False, + help="Ingestion end date for a range of file timestamps in YYYYMMDD format", + ) query_parser.add_argument( "--repointing", type=int, required=False, help="Repointing number (int)" ) From 3b8d8a6fc6116ce3072ffeabdec020f41b29920f Mon Sep 17 00:00:00 2001 From: Daralynn Rhode Date: Mon, 31 Mar 2025 14:49:03 -0600 Subject: [PATCH 2/8] ingestion added to io.py --- imap_data_access/io.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/imap_data_access/io.py b/imap_data_access/io.py index bee35b1..e8e5308 100644 --- a/imap_data_access/io.py +++ b/imap_data_access/io.py @@ -108,6 +108,8 @@ def query( descriptor: Optional[str] = None, start_date: Optional[str] = None, end_date: Optional[str] = None, + ingestion_start_date: Optional[str] = None, + ingestion_end_date: Optional[str] = None, repointing: Optional[str] = None, version: Optional[str] = None, extension: Optional[str] = None, @@ -133,6 +135,12 @@ def query( end_date : str, optional End date in YYYYMMDD format. Note this is to search for all files with start dates before the requested end_date. + ingestion_start_date : str, optional + Ingestion start date in YYYYMMDD format. Note this is to search + for all files with ingestion start dates on or after this value. + ingestion_end_date : str, optional + Ingestion end date in YYYYMMDD format. Note this is to search + for all files with ingestion start dates before the requested end_date. repointing : str, optional Repointing string, in the format 'repoint00000' version : str, optional @@ -189,6 +197,20 @@ def query( ): raise ValueError("Not a valid end date, use format 'YYYYMMDD'.") + # Check ingestion-start-date + if ( + ingestion_start_date is not None + and not file_validation.ImapFilePath.is_valid_date(ingestion_start_date) + ): + raise ValueError("Not a valid ingestion start date, use format 'YYYYMMDD'.") + + # Check ingestion-end-date + if ( + ingestion_end_date is not None + and not file_validation.ImapFilePath.is_valid_date(ingestion_end_date) + ): + raise ValueError("Not a valid ingestion end date, use format 'YYYYMMDD'.") + # Check version make sure to include 'latest' if version is not None and not file_validation.ImapFilePath.is_valid_version( version From 8fab3d6b560e17ff2a794f93d35e9e8bc0b9c815 Mon Sep 17 00:00:00 2001 From: Daralynn Rhode Date: Mon, 31 Mar 2025 14:57:35 -0600 Subject: [PATCH 3/8] updating io tests for ingestion --- tests/test_io.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_io.py b/tests/test_io.py index 26fa56c..da69826 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -178,6 +178,8 @@ def test_download_already_exists(mock_urlopen: unittest.mock.MagicMock): "descriptor": "test-description", "start_date": "20100101", "end_date": "20100102", + "ingestion_start_date": "", # Todo: Fill in + "ingestion_end_date": "", # Todo: Fill in "repointing": "repoint00001", "version": "v000", "extension": "pkts", @@ -256,6 +258,16 @@ def test_query_bad_params(mock_urlopen: unittest.mock.MagicMock): ), ("start_date", "badInput", "Not a valid start date, use format 'YYYYMMDD'."), ("end_date", "badInput", "Not a valid end date, use format 'YYYYMMDD'."), + ( + "ingestion_start_date", + "badInput", + "Not a valid ingestion start date, use format 'YYYYMMDD'.", + ), + ( + "ingestion_end_date", + "badInput", + "Not a valid ingestion end date, use format 'YYYYMMDD'.", + ), ( "repointing", "badInput", From 6ed02e88904be4e585338dcc0000d693d95a8b9f Mon Sep 17 00:00:00 2001 From: Daralynn Rhode Date: Mon, 14 Apr 2025 14:41:08 -0600 Subject: [PATCH 4/8] updating ingestion test params" --- tests/test_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_io.py b/tests/test_io.py index da69826..fd62798 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -178,8 +178,8 @@ def test_download_already_exists(mock_urlopen: unittest.mock.MagicMock): "descriptor": "test-description", "start_date": "20100101", "end_date": "20100102", - "ingestion_start_date": "", # Todo: Fill in - "ingestion_end_date": "", # Todo: Fill in + "ingestion_start_date": "20100101", # Todo: Fill in + "ingestion_end_date": "20100102", # Todo: Fill in "repointing": "repoint00001", "version": "v000", "extension": "pkts", From de24c0885c1653fb9c8a35b551a612e43edc551d Mon Sep 17 00:00:00 2001 From: Daralynn Rhode Date: Mon, 14 Apr 2025 14:48:21 -0600 Subject: [PATCH 5/8] testing param change --- tests/test_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_io.py b/tests/test_io.py index fd62798..fc3a996 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -178,8 +178,8 @@ def test_download_already_exists(mock_urlopen: unittest.mock.MagicMock): "descriptor": "test-description", "start_date": "20100101", "end_date": "20100102", - "ingestion_start_date": "20100101", # Todo: Fill in - "ingestion_end_date": "20100102", # Todo: Fill in + "ingestion_start_date": "20000101", # Todo: Fill in + "ingestion_end_date": "20000102", # Todo: Fill in "repointing": "repoint00001", "version": "v000", "extension": "pkts", From 0cd6514325c0bb326fe69628cebda3f3aed4ea94 Mon Sep 17 00:00:00 2001 From: Daralynn Rhode Date: Mon, 14 Apr 2025 15:43:46 -0600 Subject: [PATCH 6/8] removing todo comments --- tests/test_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_io.py b/tests/test_io.py index fc3a996..c054ea7 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -178,8 +178,8 @@ def test_download_already_exists(mock_urlopen: unittest.mock.MagicMock): "descriptor": "test-description", "start_date": "20100101", "end_date": "20100102", - "ingestion_start_date": "20000101", # Todo: Fill in - "ingestion_end_date": "20000102", # Todo: Fill in + "ingestion_start_date": "20100101", + "ingestion_end_date": "20100102", "repointing": "repoint00001", "version": "v000", "extension": "pkts", From 068a4e7aaa374349287ab45b8f3451549d5924b4 Mon Sep 17 00:00:00 2001 From: Daralynn Rhode Date: Tue, 13 May 2025 08:06:31 -0600 Subject: [PATCH 7/8] pre-commit reset --- imap_data_access/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_data_access/io.py b/imap_data_access/io.py index 2db1e45..edf8d87 100644 --- a/imap_data_access/io.py +++ b/imap_data_access/io.py @@ -126,7 +126,7 @@ def query( Ingestion end date in YYYYMMDD format. Note this is to search for all files with ingestion start dates before the requested end_date. repointing : str, optional - Repointing string, in the format 'repoint00000' + Repointing string, in the format 'repoint00000'. version : str, optional Data version in the format ``vXXX`` or 'latest'. extension : str, optional From 95be1fe36f6eda3fa3e504df6228d8dd15898870 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 13 May 2025 14:09:29 +0000 Subject: [PATCH 8/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- imap_data_access/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/imap_data_access/cli.py b/imap_data_access/cli.py index b44d089..0e722b2 100644 --- a/imap_data_access/cli.py +++ b/imap_data_access/cli.py @@ -355,7 +355,6 @@ def main(): # noqa: PLR0915 type=str, required=False, help="Repointing number (repoint00000)", - ) query_parser.add_argument( "--version",