diff --git a/doc/requirements.txt b/doc/requirements.txt index 133aa04ce..61b41daf8 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,7 +1,7 @@ -lxml=4.9.2 +lxml==4.9.2 requests==2.28.1 requests-cache==0.9.7 -pandas==1.5.2 +pydantic==2.5.2 +pandas==2.1.3 sphinx==5.3 ipython==8.7.0 - diff --git a/pandasdmx/api.py b/pandasdmx/api.py index 81439d5b8..37b004768 100644 --- a/pandasdmx/api.py +++ b/pandasdmx/api.py @@ -290,7 +290,8 @@ def _request_from_args(self, kwargs): url_parts.append(key) # Assemble final URL - url = "/".join(filter(None, url_parts)) + url_parts = [str(x) for x in filter(None, url_parts)] + url = "/".join(url_parts) # Parameters: set 'references' to sensible defaults if "references" not in parameters: diff --git a/pandasdmx/message.py b/pandasdmx/message.py index 609ad5ff4..9c39cbc6d 100644 --- a/pandasdmx/message.py +++ b/pandasdmx/message.py @@ -213,7 +213,8 @@ def add(self, obj: model.IdentifiableArtefact): for field, field_info in direct_fields(self.__class__).items(): # NB for some reason mypy complains here, but not in __contains__(), below if isinstance( - obj, get_args(field_info.outer_type_)[1], # type: ignore [attr-defined] + obj, + get_args(field_info.outer_type_)[1], # type: ignore [attr-defined] ): getattr(self, field)[obj.id] = obj return diff --git a/pandasdmx/model.py b/pandasdmx/model.py index 672c4d62c..5435e03fa 100644 --- a/pandasdmx/model.py +++ b/pandasdmx/model.py @@ -33,6 +33,7 @@ from operator import attrgetter, itemgetter from typing import ( Any, + ClassVar, Dict, Generator, Generic, @@ -49,6 +50,10 @@ ) from warnings import warn +from pydantic import GetCoreSchemaHandler +from pydantic_core import CoreSchema +from pydantic_core import core_schema + from pandasdmx.util import ( BaseModel, DictLike, @@ -186,18 +191,28 @@ def __eq__(self, other): return NotImplemented @classmethod - def __get_validators__(cls): - yield cls.__validate + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: + return core_schema.chain_schema( + [ + core_schema.with_info_plain_validator_function( + function=cls.__validate, + ), + ] + ) @classmethod - def __validate(cls, value, values, config, field): - # Any value that the constructor can handle can be assigned + def __validate(cls, value, info): + # Any value except None that the constructor can handle can be assigned + if value == None: + raise ValueError if not isinstance(value, InternationalString): value = InternationalString(value) try: # Update existing value - existing = values[field.name] + existing = info.data[info.field_name] existing.localizations.update(value.localizations) return existing except KeyError: @@ -602,7 +617,7 @@ class ItemScheme(MaintainableArtefact, Generic[IT]): # TODO add delete() # TODO add sorting capability; perhaps sort when new items are inserted - is_partial: Optional[bool] + is_partial: Optional[bool] = None #: Members of the ItemScheme. Both ItemScheme and Item are abstract classes. #: Concrete classes are paired: for example, a :class:`.Codelist` contains @@ -734,7 +749,7 @@ def setdefault(self, obj=None, **kwargs) -> IT: kwargs["parent"] = self[parent] # Instantiate an object of the correct class - obj = self._Item(**kwargs) + obj = self.__class__._Item.get_default()(**kwargs) try: # Add the object to the ItemScheme @@ -745,9 +760,6 @@ def setdefault(self, obj=None, **kwargs) -> IT: return obj -Item.update_forward_refs() - - # §3.6: Structure @@ -863,7 +875,7 @@ class ComponentList(IdentifiableArtefact, Generic[CT]): #: components: List[CT] = [] #: - auto_order = 1 + auto_order: ClassVar[int] = 1 # The default type of the Components in the ComponentList. See comment on # ItemScheme._Item @@ -916,7 +928,7 @@ def getdefault(self, id, cls=None, **kwargs) -> CT: # order property try: component.order = self.auto_order - self.auto_order += 1 + self.__class__.auto_order += 1 except ValueError: pass @@ -1040,7 +1052,7 @@ class Agency(Organisation): # Update forward references to 'Agency' for cls in list(locals().values()): if isclass(cls) and issubclass(cls, MaintainableArtefact): - cls.update_forward_refs() + cls.model_rebuild() class OrganisationScheme: @@ -1464,8 +1476,8 @@ def assign_order(self): pass -DimensionRelationship.update_forward_refs() -GroupRelationship.update_forward_refs() +DimensionRelationship.model_rebuild() +GroupRelationship.model_rebuild() class _NullConstraintClass: @@ -1789,7 +1801,7 @@ class KeyValue(BaseModel): #: The actual value. value: Any #: - value_for: Optional[Dimension] = None + value_for: Optional[DimensionComponent] = None def __init__(self, *args, **kwargs): args, kwargs = value_for_dsd_ref("dimension", args, kwargs) @@ -1810,6 +1822,9 @@ def __eq__(self, other): else: return self.value == other + def __lt__(self, other): + return self.value < other.value + def __str__(self): return "{0.id}={0.value}".format(self) @@ -2302,6 +2317,9 @@ class ProvisionAgreement(MaintainableArtefact, ConstrainableArtefact): data_provider: Optional[DataProvider] = None +Item.model_rebuild() + + #: The SDMX-IM defines 'packages'; these are used in URNs. PACKAGE = dict() diff --git a/pandasdmx/reader/sdmxml.py b/pandasdmx/reader/sdmxml.py index cdfc57039..7766e919a 100644 --- a/pandasdmx/reader/sdmxml.py +++ b/pandasdmx/reader/sdmxml.py @@ -119,6 +119,7 @@ class _NoText: # Sentinel value for XML elements with no text; used to distinguish from "" and None NoText = _NoText() + class Reference: """Temporary class for references. @@ -205,7 +206,7 @@ def __str__(self): # pragma: no cover class XSDResolver(etree.Resolver): """ - Resolve XSD imports to locate them within /pandaSDMX/sdmx_2_1. + Resolve XSD imports to locate them within /pandaSDMX/sdmx_2_1. """ def __init__(self, *args, schema_dir=None, **kwargs): @@ -249,7 +250,7 @@ def validate_message(msg, schema_dir=None): must be installed first. See the docs on :func:`pandasdmx.api.install_schemas` and :meth:`pandasdmx.api.Request.validate`. - + Returns whatever lxml.etree.XMLSchema.validate returns """ msg_doc = etree.parse(msg) @@ -829,7 +830,7 @@ def _ref(reader, elem): @end("com:Annotation") def _a(reader, elem): - url=reader.pop_single("AnnotationURL") + url = reader.pop_single("AnnotationURL") args = dict( title=reader.pop_single("AnnotationTitle"), type=reader.pop_single("AnnotationType"), @@ -912,7 +913,7 @@ def _itemscheme(reader, elem): is_ = reader.maintainable(cls, elem) # Iterate over all Item objects *and* their children - iter_all = chain(*[iter(item) for item in reader.pop_all(cls._Item)]) + iter_all = chain(*[iter(item) for item in reader.pop_all(cls._Item.default)]) # Set of objects already added to `items` seen = dict() diff --git a/pandasdmx/source/__init__.py b/pandasdmx/source/__init__.py index 74b9a5eda..c08cf6b9e 100644 --- a/pandasdmx/source/__init__.py +++ b/pandasdmx/source/__init__.py @@ -31,7 +31,7 @@ class Source(BaseModel): id: str #: Optional API IDTakes precedence over id when URL is constructed # Useful if a provider offers several APIs - api_id: Optional[str] + api_id: Optional[str] = None #: Base URL for queries url: Optional[HttpUrl] @@ -40,7 +40,7 @@ class Source(BaseModel): name: str #: documentation URL of the data source - documentation: Optional[HttpUrl] + documentation: Optional[HttpUrl] = None headers: Dict[str, Any] = {} @@ -124,7 +124,7 @@ def modify_request_args(self, kwargs): @validator("id") def _validate_id(cls, value): - assert getattr(cls, "_id", value) == value + assert cls.__dict__.get("_id", value) == value return value @validator("data_content_type", pre=True) diff --git a/pandasdmx/tests/test_model.py b/pandasdmx/tests/test_model.py index 6254d97d1..7c135c63c 100644 --- a/pandasdmx/tests/test_model.py +++ b/pandasdmx/tests/test_model.py @@ -3,6 +3,7 @@ import pydantic import pytest from pytest import raises +import re from pandasdmx import model from pandasdmx.model import ( @@ -199,7 +200,7 @@ def test_internationalstring(): assert str(i2.name) == "European Central Bank" # Creating with name=None raises an exception… - with raises(pydantic.ValidationError, match="none is not an allowed value"): + with raises(pydantic.ValidationError, match=re.compile(r"name\n.*input_value=None")): Item(id="ECB", name=None) # …giving empty dict is equivalent to giving nothing diff --git a/pandasdmx/util.py b/pandasdmx/util.py index d4f662058..979cd2354 100644 --- a/pandasdmx/util.py +++ b/pandasdmx/util.py @@ -8,8 +8,8 @@ import pydantic import requests from pydantic import Field, ValidationError, validator -from pydantic.class_validators import make_generic_validator -from pydantic.typing import get_origin # type: ignore [attr-defined] +from pydantic import validator +from typing import get_origin # type: ignore [attr-defined] try: import requests_cache @@ -162,7 +162,7 @@ class BaseModel(pydantic.BaseModel): """Common settings for :class:`pydantic.BaseModel` in :mod:`pandasdmx`.""" class Config: - copy_on_model_validation = 'none' + copy_on_model_validation = "none" validate_assignment = True @@ -245,14 +245,17 @@ def __get_validators__(cls): yield cls._validate_whole @classmethod - def _validate_whole(cls, v, field: pydantic.fields.ModelField): + def _validate_whole(cls, v, field: str): """Validate `v` as an entire DictLike object.""" # Convert anything that can be converted to a dict(). pydantic internals catch # most other invalid types, e.g. set(); no need to handle them here. + if cls == DictLike: + return v + result = cls(v) # Reference to the pydantic.field.ModelField for the entries - result.__field = field + result.__field = cls.model_fields[field] return result @@ -260,7 +263,7 @@ def _validate_entry(self, key, value): """Validate one `key`/`value` pair.""" try: # Use pydantic's validation machinery - v, error = self.__field._validate_mapping_like( + v, error = self.__class.model_fields[self.__field]._validate_mapping_like( ((key, value),), values={}, loc=(), cls=None ) except AttributeError: @@ -332,11 +335,11 @@ def validate_dictlike(cls): lambda item: get_origin(item[1]) is DictLike, cls.__annotations__.items() ): # Add the validator(s) - field = cls.__fields__[name] - field.post_validators = field.post_validators or [] - field.post_validators.extend( - make_generic_validator(v) for v in DictLike.__get_validators__() - ) + for v in DictLike.__get_validators__(): + + @validator(name, allow_reuse=True, pre=False) + def _validator(cls, value): + return v(cls, value, name, None) return cls @@ -388,7 +391,7 @@ def parse_content_type(value: str) -> Tuple[str, Dict[str, Any]]: @lru_cache() -def direct_fields(cls) -> Mapping[str, pydantic.fields.ModelField]: +def direct_fields(cls) -> Mapping[str, str]: """Return the :mod:`pydantic` fields defined on `obj` or its class. This is like the ``__fields__`` attribute, but excludes the fields defined on any @@ -396,8 +399,8 @@ def direct_fields(cls) -> Mapping[str, pydantic.fields.ModelField]: """ return { name: info - for name, info in cls.__fields__.items() - if name not in set(cls.mro()[1].__fields__.keys()) + for name, info in cls.model_fields.items() + if name not in set(cls.mro()[1].model_fields.keys()) } diff --git a/pyproject.toml b/pyproject.toml index de1ed28df..633d63047 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,36 +10,36 @@ home-page = "https://github.com/dr-leo/pandasdmx" dist-name = "pandaSDMX" description-file = "README.rst" requires = [ - "requests >=2.26", - "lxml >= 4.8", - "pandas >= 1.3", - "pydantic >=1.9.2, < 2.0"] + "requests >=2.26", + "lxml >= 4.8", + "pandas >= 1.3", + "pydantic >=2.0.0, <3.0", +] requires-python = ">=3.9.6,<3.12" keywords = "statistics, SDMX, pandas, data, economics, science" classifiers = [ -"Intended Audience :: Developers", -"Intended Audience :: Science/Research", - "Intended Audience :: Financial and Insurance Industry", -"Development Status :: 4 - Beta", -"Operating System :: OS Independent", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Information Analysis", -"License :: OSI Approved :: Apache Software License"] + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Intended Audience :: Financial and Insurance Industry", + "Development Status :: 4 - Beta", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Information Analysis", + "License :: OSI Approved :: Apache Software License", +] -[tool.flit.metadata.requires-extra] +[tool.flit.metadata.requires-extra] cache = ["requests_cache >= 0.9.5"] schema = ["appdirs >= 1.4"] -doc = ["sphinx >= 5.2", -"IPython >= 7.20"] -test = ["pytest >= 5", -"requests-mock >= 1.4"] +doc = ["sphinx >= 5.2", "IPython >= 7.20"] +test = ["pytest >= 5", "requests-mock >= 1.4"] [tool.flit.metadata.urls] - Homepage = "https://pandasdmx.readthedocs.io/en/latest/" +Homepage = "https://pandasdmx.readthedocs.io/en/latest/" [tool.flit.sdist] include = ["LICENSE", 'README.rst'] -exclude = ['pandasdmx/tests'] \ No newline at end of file +exclude = ['pandasdmx/tests']