From 24875ec74263e46d41ea8d2dc16ff79912b09939 Mon Sep 17 00:00:00 2001 From: AliceBalfanz Date: Wed, 9 Sep 2020 14:51:44 +0200 Subject: [PATCH 1/4] Fixed problem with unsupported time format for permafrost datasets #944 --- CHANGES.md | 1 + cate/core/opimpl.py | 6 +++--- cate/util/time.py | 3 +++ tests/ds/test_esa_cci_odp.py | 10 ++++++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0402458ee..69afb17f8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ ## Version 2.1.1 +* Fixed problem with unsupported time format for permafrost datasets #944 * Prevent HTTP 500 errors when using the ODP Data Store. #937 * Spatial points are now parsed from CSV files when using the `read_csv()`operation. This is an option which can be disabled. #935 diff --git a/cate/core/opimpl.py b/cate/core/opimpl.py index b91fe1dc4..faa3a1d46 100644 --- a/cate/core/opimpl.py +++ b/cate/core/opimpl.py @@ -33,7 +33,7 @@ from .types import PolygonLike, ValidationError from ..util.misc import to_list from ..util.monitor import Monitor -from cate.util.time import get_timestamps_from_string +from cate.util.time import get_timestamps_from_string, get_timestamp_from_string __author__ = "Janis Gailis (S[&]T Norway)" \ "Norman Fomferra (Brockmann Consult GmbH)" @@ -354,7 +354,7 @@ def _get_time_coverage_from_ds(ds: xr.Dataset) -> (pd.Timestamp, pd.Timestamp): if time_coverage_start is not None: # noinspection PyBroadException try: - time_coverage_start = pd.to_datetime(time_coverage_start) + time_coverage_start = get_timestamp_from_string(time_coverage_start) except BaseException: pass @@ -362,7 +362,7 @@ def _get_time_coverage_from_ds(ds: xr.Dataset) -> (pd.Timestamp, pd.Timestamp): if time_coverage_end is not None: # noinspection PyBroadException try: - time_coverage_end = pd.to_datetime(time_coverage_end) + time_coverage_end = get_timestamp_from_string(time_coverage_end) except BaseException: pass if time_coverage_start or time_coverage_end: diff --git a/cate/util/time.py b/cate/util/time.py index b7db00064..75f820927 100644 --- a/cate/util/time.py +++ b/cate/util/time.py @@ -31,6 +31,7 @@ (re.compile(6 * '\\d'), '%Y%m'), (re.compile(4 * '\\d'), '%Y')] + def find_datetime_format(filename: str) -> Tuple[Optional[str], int, int]: for regex, time_format in _RE_TO_DATETIME_FORMATS: searcher = regex.search(filename) @@ -39,6 +40,7 @@ def find_datetime_format(filename: str) -> Tuple[Optional[str], int, int]: return time_format, p1, p2 return None, -1, -1 + def get_timestamp_from_string(string: str) -> pd.Timestamp: time_format, p1, p2 = find_datetime_format(string) if not time_format: @@ -48,6 +50,7 @@ def get_timestamp_from_string(string: str) -> pd.Timestamp: except tslibs.OutOfBoundsDatetime: return None + def get_timestamps_from_string(string: str) -> (pd.Timestamp, pd.Timestamp): first_time = None second_time = None diff --git a/tests/ds/test_esa_cci_odp.py b/tests/ds/test_esa_cci_odp.py index ebd9e8c44..4a5ea0acc 100644 --- a/tests/ds/test_esa_cci_odp.py +++ b/tests/ds/test_esa_cci_odp.py @@ -656,3 +656,13 @@ def test_make_local_wo_subsets(self): ds = data_source.make_local(random_string) self.assertIsNotNone(ds) local_data_store.remove_data_source(f"local.{random_string}") + + +@unittest.skip(reason='Used for debugging to fix Cate issue #944') +class TestUnsupportedOperandType(unittest.TestCase): + def test_OC_unconverted_time(self): + data_store = EsaCciOdpDataStore() + cci_dataset_collection = 'esacci.PERMAFROST.yr.L4.ALT.multi-sensor.multi-platform.MODIS.01-0.r1' + data_source = data_store.query(cci_dataset_collection)[0] + ds = data_source.open_dataset(time_range=['2010-01-01', '2011-01-30'], var_names=['ALT']) + self.assertIsNotNone(ds) From 928108b67885aa5003b673b6f0d6961101fa27bc Mon Sep 17 00:00:00 2001 From: AliceBalfanz Date: Wed, 9 Sep 2020 15:01:55 +0200 Subject: [PATCH 2/4] fixing_typos --- tests/ds/test_esa_cci_odp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ds/test_esa_cci_odp.py b/tests/ds/test_esa_cci_odp.py index 4a5ea0acc..11a32da94 100644 --- a/tests/ds/test_esa_cci_odp.py +++ b/tests/ds/test_esa_cci_odp.py @@ -659,8 +659,8 @@ def test_make_local_wo_subsets(self): @unittest.skip(reason='Used for debugging to fix Cate issue #944') -class TestUnsupportedOperandType(unittest.TestCase): - def test_OC_unconverted_time(self): +class UnsupportedOperandTypeTest(unittest.TestCase): + def test_unsupported_operand_type_fix(self): data_store = EsaCciOdpDataStore() cci_dataset_collection = 'esacci.PERMAFROST.yr.L4.ALT.multi-sensor.multi-platform.MODIS.01-0.r1' data_source = data_store.query(cci_dataset_collection)[0] From d9895bd9216ad69b4b6fec58f3d53c1b6222b4e3 Mon Sep 17 00:00:00 2001 From: AliceBalfanz Date: Thu, 10 Sep 2020 08:22:41 +0200 Subject: [PATCH 3/4] removed try and except based on Tonios comment --- cate/core/opimpl.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/cate/core/opimpl.py b/cate/core/opimpl.py index faa3a1d46..f703b9a99 100644 --- a/cate/core/opimpl.py +++ b/cate/core/opimpl.py @@ -352,21 +352,15 @@ def normalize_coord_vars(ds: xr.Dataset) -> xr.Dataset: def _get_time_coverage_from_ds(ds: xr.Dataset) -> (pd.Timestamp, pd.Timestamp): time_coverage_start = ds.attrs.get('time_coverage_start') if time_coverage_start is not None: - # noinspection PyBroadException - try: - time_coverage_start = get_timestamp_from_string(time_coverage_start) - except BaseException: - pass + time_coverage_start = get_timestamp_from_string(time_coverage_start) time_coverage_end = ds.attrs.get('time_coverage_end') if time_coverage_end is not None: - # noinspection PyBroadException - try: - time_coverage_end = get_timestamp_from_string(time_coverage_end) - except BaseException: - pass + time_coverage_end = get_timestamp_from_string(time_coverage_end) + if time_coverage_start or time_coverage_end: return time_coverage_start, time_coverage_end + filename = ds.encoding.get('source', '').split('/')[-1] return get_timestamps_from_string(filename) From 2fc5fb32abd3217accbe16c43ac504347409df1e Mon Sep 17 00:00:00 2001 From: AliceBalfanz Date: Fri, 11 Sep 2020 09:28:22 +0200 Subject: [PATCH 4/4] Added more information to changelog based on formans comment --- CHANGES.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 69afb17f8..7fef9207b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,10 @@ ## Version 2.1.1 -* Fixed problem with unsupported time format for permafrost datasets #944 +* Fixed problem with unsupported time format for permafrost datasets below. They have a time_coverage_start and + time_coverage_end with a datetime format of 15 characters (#944): + * esacci.PERMAFROST.yr.L4.ALT.multi-sensor.multi-platform.MODIS.01-0.r1 + * esacci.PERMAFROST.yr.L4.GTD.multi-sensor.multi-platform.MODIS.01-0.r1 + * esacci.PERMAFROST.yr.L4.PFR.multi-sensor.multi-platform.MODIS.01-0.r1 * Prevent HTTP 500 errors when using the ODP Data Store. #937 * Spatial points are now parsed from CSV files when using the `read_csv()`operation. This is an option which can be disabled. #935