Add: get entity list should ignore bad entity if the validation is failing

CHANGELOG.md

@@ -2,6 +2,7 @@
 - add: Tutorial for saving live timeseries data (e.g., forecast) in context broker and timeseries database ([#336](https://github.com/RWTH-EBC/FiLiP/pull/363))
 - add: Tutorial for using session object ([#370](https://github.com/RWTH-EBC/FiLiP/pull/370))
 - fix: Addition of trailing slash if missing from base url ([#371](https://github.com/RWTH-EBC/FiLiP/pull/371))
+- update: omit invalid entities in `get_entity_list` ([#375](https://github.com/RWTH-EBC/FiLiP/pull/375))
 
 ### v0.6.0
 - add: Tutorial for connecting with secured endpoints ([#319](https://github.com/RWTH-EBC/FiLiP/pull/319))

filip/clients/ngsi_v2/cb.py

@@ -9,7 +9,7 @@
 from enum import Enum
 from math import inf
 from pkg_resources import parse_version
-from pydantic import PositiveInt, PositiveFloat, AnyHttpUrl
+from pydantic import PositiveInt, PositiveFloat, AnyHttpUrl, ValidationError
 from pydantic.type_adapter import TypeAdapter
 from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union
 import re
@@ -31,6 +31,8 @@
     Query,
     Update,
     PropertyFormat,
+    ContextEntityList,
+    ContextEntityKeyValuesList,
 )
 from filip.models.ngsi_v2.base import AttrsFormat
 from filip.models.ngsi_v2.subscriptions import Subscription, Message
@@ -408,13 +410,12 @@ def get_entity_list(
                 headers=headers,
             )
             if AttrsFormat.NORMALIZED in response_format:
-                adapter = TypeAdapter(List[ContextEntity])
-                return adapter.validate_python(items)
-            if AttrsFormat.KEY_VALUES in response_format:
-                adapter = TypeAdapter(List[ContextEntityKeyValues])
-                return adapter.validate_python(items)
-            return items
-
+                return ContextEntityList.model_validate({"entities": items}).entities
+            elif AttrsFormat.KEY_VALUES in response_format:
+                return ContextEntityKeyValuesList.model_validate(
+                    {"entities": items}
+                ).entities
+            return items  # in case of VALUES as response_format
         except requests.RequestException as err:
             msg = "Could not load entities"
             self.log_error(err=err, msg=msg)

filip/models/ngsi_v2/context.py

@@ -8,7 +8,7 @@
 from aenum import Enum
 from pydantic import field_validator, ConfigDict, BaseModel, Field, model_validator
 from pydantic_core.core_schema import ValidationInfo
-
+from pydantic.types import OnErrorOmit
 from filip.models.ngsi_v2.base import (
     EntityPattern,
     Expression,
@@ -624,6 +624,22 @@ def get_command_triple(
         return command, command_status, command_info
 
 
+class ContextEntityList(BaseModel):
+    """
+    Collection model for a list of context entities
+    """
+
+    entities: List[OnErrorOmit[ContextEntity]]
+
+
+class ContextEntityKeyValuesList(BaseModel):
+    """
+    Collection model for a list of context entities in key-values format
+    """
+
+    entities: List[OnErrorOmit[ContextEntityKeyValues]]
+
+
 class Query(BaseModel):
     """
     Model for queries

setup.py

@@ -13,7 +13,7 @@
     "datamodel_code_generator[http]~=0.25.0",
     "paho-mqtt~=2.0.0",
     "pandas_datapackage_reader~=0.18.0",
-    "pydantic>=2.5.2,<2.7.0",
+    "pydantic>=2.6.0,<2.7.0",
     "pydantic-settings>=2.0.0,<2.3.0",
     "geojson_pydantic~=1.0.2",
     "stringcase>=1.2.0",

tests/clients/test_ngsi_v2_cb.py

@@ -270,6 +270,52 @@ def test_entity_operations(self):
             client.post_entity(entity=res_entity, update=True)
             self.assertEqual(client.get_entity(entity_id=self.entity.id), res_entity)
 
+    @clean_test(
+        fiware_service=settings.FIWARE_SERVICE,
+        fiware_servicepath=settings.FIWARE_SERVICEPATH,
+        cb_url=settings.CB_URL,
+    )
+    def test_get_entity_list_with_bad_validation(self):
+
+        def post_entity_request(data):
+            headers = {"Content-Type": "application/json"}
+            headers.update(self.fiware_header)
+            url = f"{settings.CB_URL}v2/entities"
+            response = requests.request("POST", url, headers=headers, data=data)
+            return response
+
+        # Send bad entities to the Context Broker
+        entity_wrong_value_type = {
+            "id": "test:weather_station_1",
+            "type": "WeatherStation",
+            "temperature": {"type": "Number", "value": "Error"},
+        }
+
+        entity_wrong_unit = {
+            "id": "test:weather_station_2",
+            "type": "WeatherStation",
+            "temperature": {
+                "type": "Number",
+                "value": 20,
+                "metadata": {"unitCode": {"type": "Text", "value": "Error"}},
+            },
+        }
+
+        for entity in [entity_wrong_value_type, entity_wrong_unit]:
+            payload = json.dumps(entity)
+            post_entity_request(data=payload)
+
+        # send dummy valid entities to the Context Broker
+        entities_valid = [
+            ContextEntity(id=f"test374:Entity:{i}", type="Test") for i in range(10)
+        ]
+        self.client.update(entities=entities_valid, action_type="append")
+
+        # The bad entities should not block the whole request
+        entities_res = self.client.get_entity_list()
+
+        self.assertEqual(len(entities_res), len(entities_valid))
+
     @clean_test(
         fiware_service=settings.FIWARE_SERVICE,
         fiware_servicepath=settings.FIWARE_SERVICEPATH,