Merge pull request #240 from networktocode/pydantic-2.0

Migration to Pydantic v2

CHANGELOG.md

@@ -1,6 +1,16 @@
 # Changelog
 
-## v1.8.0 - 2023-04-18
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+## [Unreleased]
+
+### Changed
+
+- **BREAKING CHANGE** #236/240 - Upgrade to Pydantic v2. 
+
+## [1.8.0] - 2023-04-18
 
 ### Added
 
@@ -13,7 +23,7 @@
 - #77/#188 - `sync_from()` and `sync_to()` now return the `Diff` that was applied.
 - #211 - Loosened `packaging` and `structlog` library dependency constraints for broader compatibility.
 
-## v1.7.0 - 2022-11-03
+## [1.7.0] - 2022-11-03
 
 ### Changed
 
@@ -31,15 +41,15 @@
 
 ### Fixed
 
-- #149 Limit redundant CI concurrency
+- #149 - Limit redundant CI concurrency
 
-## v1.6.0 - 2022-07-09
+## [1.6.0] - 2022-07-09
 
 ### Changed
 
 - #120 - Dropped support for Python 3.6, new minimum is Python 3.7
 
-## v1.5.1 - 2022-06-30
+## [1.5.1] - 2022-06-30
 
 ### Added
 
@@ -54,13 +64,13 @@
 - #115 - Fixed ReadTheDocs rendering pipeline
 - #118 - Fixed a regression in `DiffSync.get(modelname, identifiers)` introduced in 1.5.0
 
-## v1.5.0 - 2022-06-07
+## [1.5.0] - 2022-06-07
 
 ### Added
 
 - #106 - Add a new, optional, backend store based in Redis
 
-## v1.4.3 - 2022-03-03
+## [1.4.3] - 2022-03-03
 
 ### Fixed
 
@@ -70,25 +80,25 @@
 
 ### Changed
 
-- #103 Update development dependencies
+- #103 - Update development dependencies
 
-## v1.4.2 - 2022-02-28
+## [1.4.2] - 2022-02-28
 
 **WARNING** - #90 inadvertently introduced a breaking API change in DiffSync 1.4.0 through 1.4.2 (#101); this change was reverted in #102 for DiffSync 1.4.3 and later. We recommend not using this release, and moving to 1.4.3 instead.
 
 ### Fixed
 
 - #100 - Added explicit dependency on `packaging`.
 
-## v1.4.1 - 2022-01-26
+## [1.4.1] - 2022-01-26
 
 **WARNING** - #90 inadvertently introduced a breaking API change in DiffSync 1.4.0 through 1.4.2 (#101); this change was reverted in #102 for DiffSync 1.4.3 and later. We recommend not using this release, and moving to 1.4.3 instead.
 
 ### Fixed
 
 - #95 - Removed optional dependencies on `sphinx`, `m2r2`, `sphinx-rtd-theme`, `toml`.
 
-## v1.4.0 - 2022-01-24
+## [1.4.0] - 2022-01-24
 
 **WARNING** - #90 inadvertently introduced a breaking API change in DiffSync 1.4.0 through 1.4.2 (#101); this change was reverted in #102 for DiffSync 1.4.3 and later. We recommend not using this release, and moving to 1.4.3 instead.
 
@@ -117,19 +127,19 @@
 - #51 - Update minimum Pydantic version due to security advisory GHSA-5jqp-qgf6-3pvh
 - #63 - Fix type in Readme
 
-## v1.3.0 - 2021-04-07
+## [1.3.0] - 2021-04-07
 
 ### Added
 
 - #48 - added optional `callback` argument to `diff_from`/`diff_to`/`sync_from`/`sync_to` for use with progress reporting.
 
-## v1.2.0 - 2020-12-08
+## [1.2.0] - 2020-12-08
 
 ### Added
 
 - #45 - minimum Python version lowered from 3.7 to 3.6, also now tested against Python 3.9.
 
-## v1.1.0 - 2020-12-01
+## [1.1.0] - 2020-12-01
 
 ### Added
 
@@ -147,6 +157,6 @@
 
 - #44 - On CRUD failure, do not generate an extraneous "success" log message in addition to the "failed" message
 
-## v1.0.0 - 2020-10-23
+## [1.0.0] - 2020-10-23
 
 Initial release

diffsync/__init__.py

@@ -18,7 +18,7 @@
 from typing import Callable, ClassVar, Dict, List, Optional, Tuple, Type, Union, Any, Set
 from typing_extensions import Self
 
-from pydantic import BaseModel, PrivateAttr
+from pydantic import ConfigDict, BaseModel, PrivateAttr
 import structlog  # type: ignore
 
 from diffsync.diff import Diff
@@ -99,31 +99,26 @@ class DiffSyncModel(BaseModel):
 
     _status_message: str = PrivateAttr("")
     """Message, if any, associated with the create/update/delete status value."""
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
-    class Config:  # pylint: disable=too-few-public-methods
-        """Pydantic class configuration."""
-
-        # Let us have a DiffSync as an instance variable even though DiffSync is not a Pydantic model itself.
-        arbitrary_types_allowed = True
-
-    def __init_subclass__(cls) -> None:
+    @classmethod
+    def __pydantic_init_subclass__(cls, **kwargs: Any) -> None:
         """Validate that the various class attribute declarations correspond to actual instance fields.
 
         Called automatically on subclass declaration.
         """
-        variables = cls.__fields__.keys()
         # Make sure that any field referenced by name actually exists on the model
         for attr in cls._identifiers:
-            if attr not in variables and not hasattr(cls, attr):
+            if attr not in cls.model_fields and not hasattr(cls, attr):
                 raise AttributeError(f"_identifiers {cls._identifiers} references missing or un-annotated attr {attr}")
         for attr in cls._shortname:
-            if attr not in variables:
+            if attr not in cls.model_fields:
                 raise AttributeError(f"_shortname {cls._shortname} references missing or un-annotated attr {attr}")
         for attr in cls._attributes:
-            if attr not in variables:
+            if attr not in cls.model_fields:
                 raise AttributeError(f"_attributes {cls._attributes} references missing or un-annotated attr {attr}")
         for attr in cls._children.values():
-            if attr not in variables:
+            if attr not in cls.model_fields:
                 raise AttributeError(f"_children {cls._children} references missing or un-annotated attr {attr}")
 
         # Any given field can only be in one of (_identifiers, _attributes, _children)
@@ -147,15 +142,15 @@ def dict(self, **kwargs: Any) -> Dict:
         """Convert this DiffSyncModel to a dict, excluding the diffsync field by default as it is not serializable."""
         if "exclude" not in kwargs:
             kwargs["exclude"] = {"diffsync"}
-        return super().dict(**kwargs)
+        return super().model_dump(**kwargs)
 
     def json(self, **kwargs: Any) -> StrType:
         """Convert this DiffSyncModel to a JSON string, excluding the diffsync field by default as it is not serializable."""
         if "exclude" not in kwargs:
             kwargs["exclude"] = {"diffsync"}
         if "exclude_defaults" not in kwargs:
             kwargs["exclude_defaults"] = True
-        return super().json(**kwargs)
+        return super().model_dump_json(**kwargs)
 
     def str(self, include_children: bool = True, indent: int = 0) -> StrType:
         """Build a detailed string representation of this DiffSyncModel and optionally its children."""
@@ -855,4 +850,4 @@ def count(self, model: Union[StrType, "DiffSyncModel", Type["DiffSyncModel"], No
 
 
 # DiffSyncModel references DiffSync and DiffSync references DiffSyncModel. Break the typing loop:
-DiffSyncModel.update_forward_refs()
+DiffSyncModel.model_rebuild()

docs/source/getting_started/index.rst

@@ -1,5 +1,5 @@
-###############
-Getting Started
-###############
+#########
+Upgrading
+#########
 
 .. mdinclude:: 01-getting-started.md

docs/source/index.rst

@@ -9,6 +9,7 @@ Welcome to DiffSync's documentation!
    getting_started/index
    core_engine/index
    examples/index
+   upgrading/index
    api/diffsync
    license/index
 

docs/source/upgrading/01-upgrading-to-2.0.md

@@ -0,0 +1,31 @@
+# Upgrading to 2.0
+
+With diffsync 2.0, there a couple of breaking changes. What they are and how to deal with them is described in this document.
+
+## Upgrade to Pydantic's major version 2
+
+A [migration guide](https://docs.pydantic.dev/latest/migration/) is available in the Pydantic documentation. Here are the key things that may apply to your usage of diffsync:
+
+- Any fields that are of type `Optional` now need to provide an explicit `None` default (you can use [bump-pydantic](https://github.com/pydantic/bump-pydantic) to deal with this automatically for the most part)
+
+```python
+from typing import Optional
+
+from diffsync import DiffSyncModel
+
+# Before
+class Person(DiffSyncModel):
+    _identifiers = ("name",)
+    _attributes = ("age",)
+
+    name: str
+    age: Optional[int]
+
+# After
+class BetterPerson(DiffSyncModel)
+    _identifiers = ("name",)
+    _attributes = ("age",)
+
+    name: str
+    age: Optional[int] = None
+```

docs/source/upgrading/index.rst

@@ -0,0 +1,5 @@
+#########
+Upgrading
+#########
+
+.. mdinclude:: 01-upgrading-to-2.0.md

examples/01-multiple-data-sources/models.py

@@ -40,8 +40,8 @@ class Device(DiffSyncModel):
     _children = {"interface": "interfaces"}
 
     name: str
-    site_name: Optional[str]  # note that this attribute is NOT included in _attributes
-    role: Optional[str]  # note that this attribute is NOT included in _attributes
+    site_name: Optional[str] = None  # note that this attribute is NOT included in _attributes
+    role: Optional[str] = None  # note that this attribute is NOT included in _attributes
     interfaces: List = []
 
 
@@ -56,4 +56,4 @@ class Interface(DiffSyncModel):
     name: str
     device_name: str
 
-    description: Optional[str]
+    description: Optional[str] = None

examples/03-remote-system/models.py

@@ -32,4 +32,4 @@ class Country(DiffSyncModel):
     slug: str
     name: str
     region: str
-    population: Optional[int]
+    population: Optional[int] = 0

examples/04-get-update-instantiate/models.py

@@ -38,8 +38,8 @@ class Device(DiffSyncModel):
     _children = {"interface": "interfaces", "site": "sites"}
 
     name: str
-    site_name: Optional[str]  # note that this attribute is NOT included in _attributes
-    role: Optional[str]  # note that this attribute is NOT included in _attributes
+    site_name: Optional[str] = None  # note that this attribute is NOT included in _attributes
+    role: Optional[str] = None  # note that this attribute is NOT included in _attributes
     interfaces: List = []
     sites: List = []
 
@@ -55,4 +55,4 @@ class Interface(DiffSyncModel):
     name: str
     device_name: str
 
-    description: Optional[str]
+    description: Optional[str] = None

examples/05-nautobot-peeringdb/models.py

@@ -21,8 +21,8 @@ class RegionModel(DiffSyncModel):
     # Data type declarations for all identifiers and attributes
     name: str
     slug: str
-    description: Optional[str]
-    parent_name: Optional[str]  # may be None
+    description: Optional[str] = None
+    parent_name: Optional[str] = None
     sites: List = []
 
     # Not in _attributes or _identifiers, hence not included in diff calculations
@@ -49,10 +49,10 @@ class SiteModel(DiffSyncModel):
     name: str
     slug: str
     status_slug: str
-    region_name: Optional[str]  # may be None
-    description: Optional[str]
-    latitude: Optional[float]
-    longitude: Optional[float]
+    region_name: Optional[str] = None
+    description: Optional[str] = None
+    latitude: Optional[float] = None
+    longitude: Optional[float] = None
 
     # Not in _attributes or _identifiers, hence not included in diff calculations
     pk: Optional[Union[UUID, int]]

examples/06-ip-prefixes/models.py

@@ -11,6 +11,6 @@ class Prefix(DiffSyncModel):
     _attributes = ("vrf", "vlan_id", "tenant")
 
     prefix: str
-    vrf: Optional[str]
-    vlan_id: Optional[int]
-    tenant: Optional[str]
+    vrf: Optional[str] = None
+    vlan_id: Optional[int] = None
+    tenant: Optional[str] = None