Merge pull request #945 from CCI-Tools/alicja-944-fix-unsuported-oper…

…and-types-error

Fixed problem with unsupported time format for permafrost datasets #944

CHANGES.md

@@ -3,6 +3,14 @@
 * Added package `s3fs` to Python environment as it is required to open Zarr datasets 
   from S3-compatible object store. #940
 * Fixed issue of harmonization of info field names of metadata (#949)
+* Fixed problem with unsupported time format for permafrost datasets below. They have a time_coverage_start and 
+time_coverage_end with a datetime format of 15 characters (#944):
+    * esacci.PERMAFROST.yr.L4.ALT.multi-sensor.multi-platform.MODIS.01-0.r1
+    * esacci.PERMAFROST.yr.L4.GTD.multi-sensor.multi-platform.MODIS.01-0.r1
+    * esacci.PERMAFROST.yr.L4.PFR.multi-sensor.multi-platform.MODIS.01-0.r1   
+* Fixed problem of not handling timezone aware times from dataset metadata. Some datasets have their time information 
+  stored in timezone aware timestamps e.g. '1997-09-03T00:00:00+00:00'. Cate now is able to get a datetime object of these 
+  timezone aware strings, and removes the timezone awareness. (#942) 
 
 ## Version 2.1.1
 
@@ -18,9 +26,6 @@
   in a workspace. #933
 * Fixed a problem that prevented reopening workspaces using 
   the Web API when they referenced external files. #930
-* Fixed problem of not handling timezone aware times from dataset metadata. Some datasets have their time information 
-  stored in timezone aware timestamps e.g. '1997-09-03T00:00:00+00:00'. Cate now is able to get a datetime object of these 
-  timezone aware strings, and removes the timezone awareness. (#942)  
 
 
 ## Version 2.1.0

cate/core/opimpl.py

@@ -33,7 +33,7 @@
 from .types import PolygonLike, ValidationError
 from ..util.misc import to_list
 from ..util.monitor import Monitor
-from cate.util.time import get_timestamps_from_string
+from cate.util.time import get_timestamps_from_string, get_timestamp_from_string
 
 __author__ = "Janis Gailis (S[&]T Norway)" \
              "Norman Fomferra (Brockmann Consult GmbH)"
@@ -352,21 +352,15 @@ def normalize_coord_vars(ds: xr.Dataset) -> xr.Dataset:
 def _get_time_coverage_from_ds(ds: xr.Dataset) -> (pd.Timestamp, pd.Timestamp):
     time_coverage_start = ds.attrs.get('time_coverage_start')
     if time_coverage_start is not None:
-        # noinspection PyBroadException
-        try:
-            time_coverage_start = pd.to_datetime(time_coverage_start)
-        except BaseException:
-            pass
+        time_coverage_start = get_timestamp_from_string(time_coverage_start)
 
     time_coverage_end = ds.attrs.get('time_coverage_end')
     if time_coverage_end is not None:
-        # noinspection PyBroadException
-        try:
-            time_coverage_end = pd.to_datetime(time_coverage_end)
-        except BaseException:
-            pass
+        time_coverage_end = get_timestamp_from_string(time_coverage_end)
+
     if time_coverage_start or time_coverage_end:
         return time_coverage_start, time_coverage_end
+
     filename = ds.encoding.get('source', '').split('/')[-1]
     return get_timestamps_from_string(filename)
 

cate/util/time.py

@@ -31,6 +31,7 @@
                                       (re.compile(6 * '\\d'), '%Y%m'),
                                       (re.compile(4 * '\\d'), '%Y')]
 
+
 def find_datetime_format(filename: str) -> Tuple[Optional[str], int, int]:
     for regex, time_format in _RE_TO_DATETIME_FORMATS:
         searcher = regex.search(filename)
@@ -39,6 +40,7 @@ def find_datetime_format(filename: str) -> Tuple[Optional[str], int, int]:
             return time_format, p1, p2
     return None, -1, -1
 
+
 def get_timestamp_from_string(string: str) -> pd.Timestamp:
     time_format, p1, p2 = find_datetime_format(string)
     if not time_format:
@@ -48,6 +50,7 @@ def get_timestamp_from_string(string: str) -> pd.Timestamp:
     except tslibs.OutOfBoundsDatetime:
         return None
 
+
 def get_timestamps_from_string(string: str) -> (pd.Timestamp, pd.Timestamp):
     first_time = None
     second_time = None

tests/ds/test_esa_cci_odp.py

@@ -681,3 +681,12 @@ def test_unconverted_time(self):
         data_source = data_store.query(cci_dataset_collection)[0]
         ds = data_source.open_dataset(time_range=['2010-01-01', '2010-01-30'], var_names=['CHLOR_A'])
         self.assertIsNotNone(ds)
+
+@unittest.skip(reason='Used for debugging to fix Cate issue #944')
+class UnsupportedOperandTypeTest(unittest.TestCase):
+    def test_unsupported_operand_type_fix(self):
+        data_store = EsaCciOdpDataStore()
+        cci_dataset_collection = 'esacci.PERMAFROST.yr.L4.ALT.multi-sensor.multi-platform.MODIS.01-0.r1'
+        data_source = data_store.query(cci_dataset_collection)[0]
+        ds = data_source.open_dataset(time_range=['2010-01-01', '2011-01-30'], var_names=['ALT'])
+        self.assertIsNotNone(ds)