Add the IDU source data validator

- Add the method to validate the source data items
- Update the idu validator
- Validate the data before assignment
- Gidd validator added
- Glide validator added
- Em dat data validation
- Gdacs data validators
- GFD data validators
- GIDD data validators
- IFRC data validators
- GLIDE data validators
- Ibtracs validator added
- Fix ibtracs collection id and country code.
- GLIDE data validators
- Add partial success for ibtrace.

Author: ranjan-stha

pyproject.toml

@@ -33,7 +33,7 @@ dependencies = [
     "pytz>=2021.1",
     "pandas>=2.2.0",
     "lxml>=5.3.0",
-    "pydantic",
+    "pydantic>=2.10.6",
 ]
 dynamic = ["version"]
 

pystac_monty/sources/emdat.py

@@ -19,6 +19,7 @@
 from pystac_monty.hazard_profiles import MontyHazardProfiles
 from pystac_monty.sources.common import MontyDataSource, MontyDataTransformer
 from pystac_monty.utils import rename_columns
+from pystac_monty.validators.em_dat import EmdatDataValidator
 
 STAC_EVENT_ID_PREFIX = "emdat-event-"
 STAC_HAZARD_ID_PREFIX = "emdat-hazard-"
@@ -40,12 +41,24 @@ def __init__(self, source_url: str, data: Union[str, pd.DataFrame]):
             self.df = data
         elif isinstance(data, dict):
             # If data is a dict, assume it's Json content
-            data = data["data"]["public_emdat"]["data"]
+            # data = data["data"]["public_emdat"]["data"]
+            data = self.source_data_validator(data["data"]["public_emdat"]["data"])
             df = pd.DataFrame(data)
             self.df = rename_columns(df)
         else:
             raise ValueError("Data must be either Excel content (str) or pandas DataFrame or Json")
 
+    def source_data_validator(self, data):
+        valid_list = []
+        error_list = []
+        for item in data:
+            if EmdatDataValidator.validate_event(item):
+                valid_list.append(item)
+            else:
+                error_list.append(item)
+
+        return valid_list
+
     def get_data(self) -> pd.DataFrame:
         return self.df
 

pystac_monty/sources/gdacs.py

@@ -22,6 +22,8 @@
 )
 from pystac_monty.hazard_profiles import MontyHazardProfiles
 from pystac_monty.sources.common import MontyDataSource, MontyDataTransformer
+from pystac_monty.validators.gdacs_events import GdacsDataValidatorEvents
+from pystac_monty.validators.gdacs_geometry import GdacsDataValidatorGeometry
 
 # Constants
 
@@ -46,7 +48,24 @@ def __init__(self, source_url: str, data: Any, type: GDACSDataSourceType):
         super().__init__(source_url, data)
         self.type = type
         # all gdacs data are json
-        self.data = json.loads(data)
+        self.data = self.source_data_validator(json.loads(data))
+
+    def source_data_validator(self, data: dict):
+        # Debug print
+        if self.type == GDACSDataSourceType.EVENT:
+            result = GdacsDataValidatorEvents.validate_event(data)
+            if result:
+                return data
+        elif self.type == GDACSDataSourceType.GEOMETRY:
+            new_data = {}  # Store the filtered dictionary
+            for key, value in data.items():
+                if key == "features" and isinstance(value, list):
+                    # Validate each feature in the list and skip the ones with 'Figure cause' = 'Conflict'
+                    new_data[key] = [feature for feature in value if GdacsDataValidatorGeometry.validate_event(feature)]
+                else:
+                    # Keep normal key-value pairs unchanged
+                    new_data[key] = value
+            return new_data
 
     def get_type(self) -> GDACSDataSourceType:
         return self.type

pystac_monty/sources/gfd.py

@@ -15,6 +15,7 @@
 )
 from pystac_monty.hazard_profiles import MontyHazardProfiles
 from pystac_monty.sources.common import MontyDataSource, MontyDataTransformer
+from pystac_monty.validators.gfd import GFDSourceValidator
 
 # Constants
 
@@ -28,7 +29,20 @@ class GFDDataSource(MontyDataSource):
 
     def __init__(self, source_url: str, data: Any):
         super().__init__(source_url, data)
-        self.data = json.loads(data)
+        self.data = self.source_data_validator(json.loads(data))
+
+    def source_data_validator(self, data: list[dict]):
+        """Validate the source data and collect only the success items"""
+        # TODO Handle the failed_items
+        failed_items = []
+        success_items = []
+        for item in data:
+            is_valid = GFDSourceValidator.validate_event(item)
+            if is_valid:
+                success_items.append(item)
+            else:
+                failed_items.append(item)
+        return success_items
 
 
 class GFDTransformer(MontyDataTransformer[GFDDataSource]):
@@ -85,7 +99,7 @@ def make_source_event_item(self, data: dict) -> Item:
         enddate = pytz.utc.localize(datetime.fromtimestamp(data["system:time_end"] / 1000))
 
         item = Item(
-            id=f'{STAC_EVENT_ID_PREFIX}{data["id"]}',
+            id=f"{STAC_EVENT_ID_PREFIX}{data['id']}",
             geometry=geometry,
             bbox=bbox,
             datetime=startdate,

pystac_monty/sources/gidd.py

@@ -17,6 +17,7 @@
 from pystac_monty.hazard_profiles import MontyHazardProfiles
 from pystac_monty.sources.common import MontyDataSource, MontyDataTransformer
 from pystac_monty.sources.utils import IDMCUtils
+from pystac_monty.validators.gidd import GiddValidator
 
 logger = logging.getLogger(__name__)
 
@@ -31,6 +32,21 @@ class GIDDDataSource(MontyDataSource):
     def __init__(self, source_url: str, data: Any):
         super().__init__(source_url, data)
         self.data = json.loads(data)
+        self.data = self.source_data_validator(json.loads(data))
+
+    def source_data_validator(self, data: dict):
+        """Validate only the items inside 'features' while keeping other keys unchanged."""
+
+        new_data = {}  # Store the filtered dictionary
+
+        for key, value in data.items():
+            if key == "features" and isinstance(value, list):
+                # Validate each feature in the list and skip the ones with 'Figure cause' = 'Conflict'
+                new_data[key] = [feature for feature in value if GiddValidator.validate_event(feature)]
+            else:
+                # Keep normal key-value pairs unchanged
+                new_data[key] = value
+        return new_data
 
 
 class GIDDTransformer(MontyDataTransformer[GIDDDataSource]):

pystac_monty/sources/glide.py

@@ -9,6 +9,7 @@
 from pystac_monty.extension import HazardDetail, MontyEstimateType, MontyExtension
 from pystac_monty.hazard_profiles import MontyHazardProfiles
 from pystac_monty.sources.common import MontyDataSource, MontyDataTransformer
+from pystac_monty.validators.glide import GlideSetValidator
 
 STAC_EVENT_ID_PREFIX = "glide-event-"
 STAC_HAZARD_ID_PREFIX = "glide-hazard-"
@@ -17,7 +18,19 @@
 class GlideDataSource(MontyDataSource):
     def __init__(self, source_url: str, data: Any):
         super().__init__(source_url, data)
-        self.data = json.loads(data)
+        self.data = self.source_data_validator(json.loads(data))
+
+    def source_data_validator(self, data: dict[dict]):
+        """Validate the source data and collect only the success items"""
+        failed_items = []
+        success_items = []
+        for item in data["glideset"]:
+            is_valid = GlideSetValidator.validate_event(item)
+            if is_valid:
+                success_items.append(item)
+            else:
+                failed_items.append(item)
+        return {"glideset": success_items}
 
 
 class GlideTransformer(MontyDataTransformer[GlideDataSource]):