CMCD-452-dont expose datasets that are retired

mercator-ocean · Mar 22, 2024 · 5b2e79f · 5b2e79f
1 parent 5374510
commit 5b2e79f
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 17 deletions.
diff --git a/copernicusmarine/catalogue_parser/catalogue_parser.py b/copernicusmarine/catalogue_parser/catalogue_parser.py
@@ -204,6 +204,7 @@ class CopernicusMarineService:
 class CopernicusMarineVersionPart:
     name: str
     services: list[CopernicusMarineService]
+    retired_date: Optional[str]
 
     def get_service_by_service_type(
         self, service_type: CopernicusMarineDatasetServiceType
@@ -296,7 +297,9 @@ def add_raw_services(
                     label=VERSION_DEFAULT,
                     parts=[
                         CopernicusMarineVersionPart(
-                            name=PART_DEFAULT, services=portal_services
+                            name=PART_DEFAULT,
+                            services=portal_services,
+                            retired_date=None,
                         )
                     ],
                 )
@@ -540,13 +543,21 @@ def _get_parts(
     parts: List[CopernicusMarineVersionPart] = []
 
     for datacube in datacubes:
+        retired_date = datacube.properties.get("admp_retired_date")
+        if retired_date and datetime_parser(retired_date) < datetime_parser(
+            "now"
+        ):
+            continue
+
         services = _get_services(datacube)
         _, _, part = get_version_and_part_from_full_dataset_id(datacube.id)
 
         if services:
             parts.append(
                 CopernicusMarineVersionPart(
-                    name=part, services=_get_services(datacube)
+                    name=part,
+                    services=_get_services(datacube),
+                    retired_date=retired_date,
                 )
             )
 
@@ -671,11 +682,12 @@ def _construct_marine_data_store_dataset(
     )
     if datacubes:
         versions = _get_versions_from_marine_datastore(datacubes)
-        return ProductDatasetFromMarineDataStore(
-            dataset_id=dataset_id,
-            dataset_name=dataset_name,
-            versions=versions,
-        )
+        if versions:
+            return ProductDatasetFromMarineDataStore(
+                dataset_id=dataset_id,
+                dataset_name=dataset_name,
+                versions=versions,
+            )
     return None
 
 
@@ -1177,26 +1189,38 @@ def to_dataset(
             distinct_dataset_versions, key=lambda x: x.dataset_version
         )
         versions = list(chain(map_reject_none(to_version, dataset_by_version)))
-        product = ProductDatasetFromPortal(
-            dataset_id=dataset_id,
-            dataset_name=sub_dataset_title,
-            versions=versions,
-            raw_services=first_distinct_dataset_version.raw_services,
-            layer_elements=first_distinct_dataset_version.layer_elements,
-        )
-        return product
+        if versions:
+            dataset = ProductDatasetFromPortal(
+                dataset_id=dataset_id,
+                dataset_name=sub_dataset_title,
+                versions=versions,
+                raw_services=first_distinct_dataset_version.raw_services,
+                layer_elements=first_distinct_dataset_version.layer_elements,
+            )
+            return dataset
     return None
 
 
 def to_part(
     distinct_dataset_version: DistinctDatasetVersionPart,
 ) -> List[CopernicusMarineVersionPart]:
     mds_stac_services = mds_stac_to_services(distinct_dataset_version)
-    if mds_stac_services:
+    retired_date = (
+        distinct_dataset_version.stac_items_values.get("properties", {}).get(
+            "admp_retired_date"
+        )
+        if distinct_dataset_version.stac_items_values
+        else None
+    )
+    if mds_stac_services and (
+        not retired_date
+        or datetime_parser(retired_date) > datetime_parser("now")
+    ):
         return [
             CopernicusMarineVersionPart(
                 name=distinct_dataset_version.dataset_part,
                 services=mds_stac_services,
+                retired_date=retired_date,
             )
         ]
     else:

diff --git a/copernicusmarine/core_functions/utils.py b/copernicusmarine/core_functions/utils.py
@@ -126,6 +126,8 @@ class WrongDatetimeFormat(Exception):
 
 
 def datetime_parser(string: str):
+    if string == "now":
+        return datetime.now()
     for format in DATETIME_SUPPORTED_FORMATS:
         try:
             return datetime.strptime(string, format)

diff --git a/tests/test_command_line_interface.py b/tests/test_command_line_interface.py
@@ -330,7 +330,7 @@ def then_I_can_read_the_json_including_datasets(self):
                         assert part["name"] is not None
                         assert part["name"] != ""
                         services = part["services"]
-                        assert len(services) != 0
+                        assert len(services) != 0, dataset["dataset_id"]
                         service_names = list(
                             map(
                                 lambda x: x["service_type"]["service_name"],