From 077a4af48b0fd17332baf046f2fbf97d7adc8ec6 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Sun, 15 Sep 2024 21:54:24 +0200 Subject: [PATCH 1/7] chore(types): re-order types per module --- openfisca_core/{types/_domain.py => types.py} | 130 ++++++++++-------- openfisca_core/types/__init__.py | 83 ----------- openfisca_core/types/_data.py | 54 -------- 3 files changed, 76 insertions(+), 191 deletions(-) rename openfisca_core/{types/_domain.py => types.py} (66%) delete mode 100644 openfisca_core/types/__init__.py delete mode 100644 openfisca_core/types/_data.py diff --git a/openfisca_core/types/_domain.py b/openfisca_core/types.py similarity index 66% rename from openfisca_core/types/_domain.py rename to openfisca_core/types.py index d324f1b2c..ede4ba9bc 100644 --- a/openfisca_core/types/_domain.py +++ b/openfisca_core/types.py @@ -1,134 +1,138 @@ from __future__ import annotations import typing_extensions -from typing import Any, Optional -from typing_extensions import Protocol +from collections.abc import Sequence +from numpy.typing import NDArray +from typing import Any, TypeVar +from typing_extensions import Protocol, TypeAlias import abc import numpy +N = TypeVar("N", bound=numpy.generic, covariant=True) -class Entity(Protocol): - """Entity protocol.""" +#: Type representing an numpy array. +Array: TypeAlias = NDArray[N] + +L = TypeVar("L") + +#: Type representing an array-like object. +ArrayLike: TypeAlias = Sequence[L] + +#: Type variable representing an error. +E = TypeVar("E", covariant=True) + +#: Type variable representing a value. +A = TypeVar("A", covariant=True) + +# Entities + + +class Entity(Protocol): key: Any plural: Any @abc.abstractmethod def check_role_validity(self, role: Any) -> None: - """Abstract method.""" + ... @abc.abstractmethod def check_variable_defined_for_entity(self, variable_name: Any) -> None: - """Abstract method.""" + ... @abc.abstractmethod def get_variable( self, variable_name: Any, check_existence: Any = ..., - ) -> Optional[Any]: - """Abstract method.""" + ) -> Any | None: + ... -class Formula(Protocol): - """Formula protocol.""" +class Role(Protocol): + entity: Any + subroles: Any - @abc.abstractmethod - def __call__( - self, - population: Population, - instant: Instant, - params: Params, - ) -> numpy.ndarray: - """Abstract method.""" +# Holders -class Holder(Protocol): - """Holder protocol.""" +class Holder(Protocol): @abc.abstractmethod def clone(self, population: Any) -> Holder: - """Abstract method.""" + ... @abc.abstractmethod def get_memory_usage(self) -> Any: - """Abstract method.""" + ... -class Instant(Protocol): - """Instant protocol.""" +# Parameters @typing_extensions.runtime_checkable class ParameterNodeAtInstant(Protocol): - """ParameterNodeAtInstant protocol.""" + ... -class Params(Protocol): - """Params protocol.""" +# Periods - @abc.abstractmethod - def __call__(self, instant: Instant) -> ParameterNodeAtInstant: - """Abstract method.""" + +class Instant(Protocol): + ... @typing_extensions.runtime_checkable class Period(Protocol): - """Period protocol.""" - @property @abc.abstractmethod def start(self) -> Any: - """Abstract method.""" + ... @property @abc.abstractmethod def unit(self) -> Any: - """Abstract method.""" + ... -class Population(Protocol): - """Population protocol.""" +# Populations + +class Population(Protocol): entity: Any @abc.abstractmethod def get_holder(self, variable_name: Any) -> Any: - """Abstract method.""" - + ... -class Role(Protocol): - """Role protocol.""" - entity: Any - subroles: Any +# Simulations class Simulation(Protocol): - """Simulation protocol.""" - @abc.abstractmethod def calculate(self, variable_name: Any, period: Any) -> Any: - """Abstract method.""" + ... @abc.abstractmethod def calculate_add(self, variable_name: Any, period: Any) -> Any: - """Abstract method.""" + ... @abc.abstractmethod def calculate_divide(self, variable_name: Any, period: Any) -> Any: - """Abstract method.""" + ... @abc.abstractmethod - def get_population(self, plural: Optional[Any]) -> Any: - """Abstract method.""" + def get_population(self, plural: Any | None) -> Any: + ... -class TaxBenefitSystem(Protocol): - """TaxBenefitSystem protocol.""" +# Tax-Benefit systems + +class TaxBenefitSystem(Protocol): person_entity: Any @abc.abstractmethod @@ -136,11 +140,29 @@ def get_variable( self, variable_name: Any, check_existence: Any = ..., - ) -> Optional[Any]: + ) -> Any | None: """Abstract method.""" -class Variable(Protocol): - """Variable protocol.""" +# Variables + +class Variable(Protocol): entity: Any + + +class Formula(Protocol): + @abc.abstractmethod + def __call__( + self, + population: Population, + instant: Instant, + params: Params, + ) -> Array[Any]: + ... + + +class Params(Protocol): + @abc.abstractmethod + def __call__(self, instant: Instant) -> ParameterNodeAtInstant: + ... diff --git a/openfisca_core/types/__init__.py b/openfisca_core/types/__init__.py deleted file mode 100644 index eb403c46c..000000000 --- a/openfisca_core/types/__init__.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Data types and protocols used by OpenFisca Core. - -The type definitions included in this sub-package are intended for -contributors, to help them better understand and document contracts -and expected behaviours. - -Official Public API: - * :attr:`.Array` - * ``ArrayLike`` - * :attr:`.Cache` - * :attr:`.Entity` - * :attr:`.Formula` - * :attr:`.Holder` - * :attr:`.Instant` - * :attr:`.ParameterNodeAtInstant` - * :attr:`.Params` - * :attr:`.Period` - * :attr:`.Population` - * :attr:`.Role`, - * :attr:`.Simulation`, - * :attr:`.TaxBenefitSystem` - * :attr:`.Variable` - -Note: - How imports are being used today:: - - from openfisca_core.types import * # Bad - from openfisca_core.types.data_types.arrays import ArrayLike # Bad - - The previous examples provoke cyclic dependency problems, that prevents us - from modularizing the different components of the library, so as to make - them easier to test and to maintain. - - How could them be used after the next major release:: - - from openfisca_core.types import ArrayLike - - ArrayLike # Good: import types as publicly exposed - - .. seealso:: `PEP8#Imports`_ and `OpenFisca's Styleguide`_. - - .. _PEP8#Imports: - https://www.python.org/dev/peps/pep-0008/#imports - - .. _OpenFisca's Styleguide: - https://github.com/openfisca/openfisca-core/blob/master/STYLEGUIDE.md - -""" - -# Official Public API - -from ._data import Array, ArrayLike # noqa: F401 -from ._domain import ( # noqa: F401 - Entity, - Formula, - Holder, - Instant, - ParameterNodeAtInstant, - Params, - Period, - Population, - Role, - Simulation, - TaxBenefitSystem, - Variable, -) - -__all__ = [ - "Array", - "ArrayLike", - "Entity", - "Formula", - "Holder", - "Instant", - "ParameterNodeAtInstant", - "Params", - "Period", - "Population", - "Role", - "Simulation", - "TaxBenefitSystem", - "Variable", -] diff --git a/openfisca_core/types/_data.py b/openfisca_core/types/_data.py deleted file mode 100644 index 928e1b917..000000000 --- a/openfisca_core/types/_data.py +++ /dev/null @@ -1,54 +0,0 @@ -# from typing import Sequence, TypeVar, Union -# from nptyping import types, NDArray as Array -from numpy.typing import NDArray as Array # noqa: F401 -from typing import Sequence, TypeVar - -# import numpy - -# NumpyT = TypeVar("NumpyT", numpy.bytes_, numpy.number, numpy.object_, numpy.str_) -T = TypeVar("T", bool, bytes, float, int, object, str) - -# types._ndarray_meta._Type = Union[type, numpy.dtype, TypeVar] - -# ArrayLike = Union[Array[T], Sequence[T]] -ArrayLike = Sequence[T] -""":obj:`typing.Generic`: Type of any castable to :class:`numpy.ndarray`. - -These include any :obj:`numpy.ndarray` and sequences (like -:obj:`list`, :obj:`tuple`, and so on). - -Examples: - >>> ArrayLike[float] - typing.Union[numpy.ndarray, typing.Sequence[float]] - - >>> ArrayLike[str] - typing.Union[numpy.ndarray, typing.Sequence[str]] - -Note: - It is possible since numpy version 1.21 to specify the type of an - array, thanks to `numpy.typing.NDArray`_:: - - from numpy.typing import NDArray - NDArray[numpy.float64] - - `mypy`_ provides `duck type compatibility`_, so an :obj:`int` is - considered to be valid whenever a :obj:`float` is expected. - -Todo: - * Refactor once numpy version >= 1.21 is used. - -.. versionadded:: 35.5.0 - -.. versionchanged:: 35.6.0 - Moved to :mod:`.types` - -.. _mypy: - https://mypy.readthedocs.io/en/stable/ - -.. _duck type compatibility: - https://mypy.readthedocs.io/en/stable/duck_type_compatibility.html - -.. _numpy.typing.NDArray: - https://numpy.org/doc/stable/reference/typing.html#numpy.typing.NDArray - -""" From a694d281fe0115eb67ce7095487b3f44e253933a Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Sun, 15 Sep 2024 22:08:35 +0200 Subject: [PATCH 2/7] fix(types): commons --- openfisca_core/commons/formulas.py | 20 ++++++++++---------- openfisca_core/commons/misc.py | 4 ++-- openfisca_core/commons/rates.py | 14 +++++++------- openfisca_tasks/lint.mk | 4 +++- setup.cfg | 9 ++++++--- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/openfisca_core/commons/formulas.py b/openfisca_core/commons/formulas.py index bbcc4fe56..1cb35a571 100644 --- a/openfisca_core/commons/formulas.py +++ b/openfisca_core/commons/formulas.py @@ -1,17 +1,15 @@ -from typing import Any, Dict, Sequence, TypeVar +from typing import Any, Dict, Union from openfisca_core.types import Array, ArrayLike import numpy -T = TypeVar("T") - def apply_thresholds( - input: Array[float], + input: Array[numpy.float_], thresholds: ArrayLike[float], choices: ArrayLike[float], -) -> Array[float]: +) -> Array[numpy.float_]: """Makes a choice based on an input and thresholds. From a list of ``choices``, this function selects one of these values @@ -40,7 +38,7 @@ def apply_thresholds( """ - condlist: Sequence[Array[bool]] + condlist: list[Union[Array[numpy.bool_], bool]] condlist = [input <= threshold for threshold in thresholds] if len(condlist) == len(choices) - 1: @@ -58,7 +56,9 @@ def apply_thresholds( return numpy.select(condlist, choices) -def concat(this: ArrayLike[str], that: ArrayLike[str]) -> Array[str]: +def concat( + this: Union[Array[Any], ArrayLike[str]], that: Union[Array[Any], ArrayLike[str]] +) -> Array[numpy.str_]: """Concatenates the values of two arrays. Args: @@ -88,8 +88,8 @@ def concat(this: ArrayLike[str], that: ArrayLike[str]) -> Array[str]: def switch( conditions: Array[Any], - value_by_condition: Dict[float, T], -) -> Array[T]: + value_by_condition: Dict[float, Any], +) -> Array[Any]: """Mimicks a switch statement. Given an array of conditions, returns an array of the same size, @@ -120,4 +120,4 @@ def switch( condlist = [conditions == condition for condition in value_by_condition.keys()] - return numpy.select(condlist, value_by_condition.values()) + return numpy.select(condlist, tuple(value_by_condition.values())) diff --git a/openfisca_core/commons/misc.py b/openfisca_core/commons/misc.py index ee985071b..9c7fcc68f 100644 --- a/openfisca_core/commons/misc.py +++ b/openfisca_core/commons/misc.py @@ -1,4 +1,4 @@ -from typing import TypeVar +from typing import Any, TypeVar, Union from openfisca_core.types import Array @@ -43,7 +43,7 @@ def empty_clone(original: T) -> T: return new -def stringify_array(array: Array) -> str: +def stringify_array(array: Union[Array[Any], None]) -> str: """Generates a clean string representation of a numpy array. Args: diff --git a/openfisca_core/commons/rates.py b/openfisca_core/commons/rates.py index 7ede496f8..6df1f1fee 100644 --- a/openfisca_core/commons/rates.py +++ b/openfisca_core/commons/rates.py @@ -6,10 +6,10 @@ def average_rate( - target: Array[float], + target: Array[numpy.float_], varying: ArrayLike[float], trim: Optional[ArrayLike[float]] = None, -) -> Array[float]: +) -> Array[numpy.float_]: """Computes the average rate of a target net income. Given a ``target`` net income, and according to the ``varying`` gross @@ -41,7 +41,7 @@ def average_rate( """ - average_rate: Array[float] + average_rate: Array[numpy.float_] average_rate = 1 - target / varying @@ -62,10 +62,10 @@ def average_rate( def marginal_rate( - target: Array[float], - varying: Array[float], + target: Array[numpy.float_], + varying: Array[numpy.float_], trim: Optional[ArrayLike[float]] = None, -) -> Array[float]: +) -> Array[numpy.float_]: """Computes the marginal rate of a target net income. Given a ``target`` net income, and according to the ``varying`` gross @@ -97,7 +97,7 @@ def marginal_rate( """ - marginal_rate: Array[float] + marginal_rate: Array[numpy.float_] marginal_rate = +1 - (target[:-1] - target[1:]) / (varying[:-1] - varying[1:]) diff --git a/openfisca_tasks/lint.mk b/openfisca_tasks/lint.mk index 324104f7c..828c51800 100644 --- a/openfisca_tasks/lint.mk +++ b/openfisca_tasks/lint.mk @@ -39,7 +39,9 @@ lint-doc-%: ## Run static type checkers for type errors. check-types: @$(call print_help,$@:) - @mypy openfisca_core/entities openfisca_core/projectors + @mypy \ + openfisca_core/commons \ + openfisca_core/types.py @$(call print_pass,$@:) ## Run code formatters to correct style errors. diff --git a/setup.cfg b/setup.cfg index 9673496d7..d54f6263d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,9 +65,12 @@ python_files = **/*.py testpaths = tests [mypy] -ignore_missing_imports = True -install_types = True -non_interactive = True +disallow_any_unimported = true +ignore_missing_imports = true +install_types = true +non_interactive = true +plugins = numpy.typing.mypy_plugin +python_version = 3.9 [mypy-openfisca_core.commons.tests.*] ignore_errors = True From 69d9a32e076edd2aceec6cc4584dea0dc1762ba9 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 17 Sep 2024 01:23:24 +0200 Subject: [PATCH 3/7] build(make): fix make test --- openfisca_tasks/test_code.mk | 3 +-- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index d84956ea5..723c94ece 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -36,8 +36,7 @@ test-core: $(shell pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 openfisca_core/entities \ openfisca_core/holders \ openfisca_core/periods \ - openfisca_core/projectors \ - openfisca_core/types + openfisca_core/projectors @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ coverage run -m \ ${openfisca} test $? \ diff --git a/setup.cfg b/setup.cfg index d54f6263d..e266be625 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ docstring_style = google extend-ignore = D ignore = E203, E501, F405, RST301, W503 in-place = true -include-in-doctest = openfisca_core/commons openfisca_core/entities openfisca_core/holders openfisca_core/periods openfisca_core/projectors openfisca_core/types +include-in-doctest = openfisca_core/commons openfisca_core/entities openfisca_core/holders openfisca_core/periods openfisca_core/projectors max-line-length = 88 per-file-ignores = */types.py:D101,D102,E704, */test_*.py:D101,D102,D103, */__init__.py:F401 rst-directives = attribute, deprecated, seealso, versionadded, versionchanged From e185a1d9e67acf181757e48076f574968ad54023 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 17 Sep 2024 01:45:18 +0200 Subject: [PATCH 4/7] chore(commons): remove outdated Dict --- openfisca_core/commons/formulas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openfisca_core/commons/formulas.py b/openfisca_core/commons/formulas.py index 1cb35a571..c1cc159e5 100644 --- a/openfisca_core/commons/formulas.py +++ b/openfisca_core/commons/formulas.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Union +from typing import Any, Union from openfisca_core.types import Array, ArrayLike @@ -88,7 +88,7 @@ def concat( def switch( conditions: Array[Any], - value_by_condition: Dict[float, Any], + value_by_condition: dict[float, Any], ) -> Array[Any]: """Mimicks a switch statement. From 32b61f2f1e4a9b6ac1f764cd9399bff0ae469970 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 17 Sep 2024 03:35:52 +0200 Subject: [PATCH 5/7] refactor(types): avoid any --- openfisca_core/commons/formulas.py | 14 ++++++++------ openfisca_core/commons/misc.py | 6 ++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/openfisca_core/commons/formulas.py b/openfisca_core/commons/formulas.py index c1cc159e5..bb4b95b5a 100644 --- a/openfisca_core/commons/formulas.py +++ b/openfisca_core/commons/formulas.py @@ -1,4 +1,5 @@ -from typing import Any, Union +from collections.abc import Mapping +from typing import Union from openfisca_core.types import Array, ArrayLike @@ -57,7 +58,8 @@ def apply_thresholds( def concat( - this: Union[Array[Any], ArrayLike[str]], that: Union[Array[Any], ArrayLike[str]] + this: Union[Array[numpy.str_], ArrayLike[str]], + that: Union[Array[numpy.str_], ArrayLike[str]], ) -> Array[numpy.str_]: """Concatenates the values of two arrays. @@ -66,7 +68,7 @@ def concat( that: Another array to concatenate. Returns: - :obj:`numpy.ndarray` of :obj:`float`: + :obj:`numpy.ndarray` of :obj:`numpy.str_`: An array with the concatenated values. Examples: @@ -87,9 +89,9 @@ def concat( def switch( - conditions: Array[Any], - value_by_condition: dict[float, Any], -) -> Array[Any]: + conditions: Array[numpy.float_], + value_by_condition: Mapping[float, float], +) -> Array[numpy.float_]: """Mimicks a switch statement. Given an array of conditions, returns an array of the same size, diff --git a/openfisca_core/commons/misc.py b/openfisca_core/commons/misc.py index 9c7fcc68f..ff1c2e386 100644 --- a/openfisca_core/commons/misc.py +++ b/openfisca_core/commons/misc.py @@ -1,7 +1,9 @@ -from typing import Any, TypeVar, Union +from typing import Optional, TypeVar from openfisca_core.types import Array +import numpy + T = TypeVar("T") @@ -43,7 +45,7 @@ def empty_clone(original: T) -> T: return new -def stringify_array(array: Union[Array[Any], None]) -> str: +def stringify_array(array: Optional[Array[numpy.generic]]) -> str: """Generates a clean string representation of a numpy array. Args: From d835de6791b98b4084901792224c20b7a6be0a15 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 17 Sep 2024 05:14:37 +0200 Subject: [PATCH 6/7] doc(commons): add py.typed --- openfisca_core/commons/formulas.py | 24 ++++++++++++------------ openfisca_core/commons/misc.py | 6 +++--- openfisca_core/commons/py.typed | 0 3 files changed, 15 insertions(+), 15 deletions(-) create mode 100644 openfisca_core/commons/py.typed diff --git a/openfisca_core/commons/formulas.py b/openfisca_core/commons/formulas.py index bb4b95b5a..bce920693 100644 --- a/openfisca_core/commons/formulas.py +++ b/openfisca_core/commons/formulas.py @@ -1,16 +1,16 @@ from collections.abc import Mapping from typing import Union -from openfisca_core.types import Array, ArrayLike - import numpy +from openfisca_core import types as t + def apply_thresholds( - input: Array[numpy.float_], - thresholds: ArrayLike[float], - choices: ArrayLike[float], -) -> Array[numpy.float_]: + input: t.Array[numpy.float_], + thresholds: t.ArrayLike[float], + choices: t.ArrayLike[float], +) -> t.Array[numpy.float_]: """Makes a choice based on an input and thresholds. From a list of ``choices``, this function selects one of these values @@ -39,7 +39,7 @@ def apply_thresholds( """ - condlist: list[Union[Array[numpy.bool_], bool]] + condlist: list[Union[t.Array[numpy.bool_], bool]] condlist = [input <= threshold for threshold in thresholds] if len(condlist) == len(choices) - 1: @@ -58,9 +58,9 @@ def apply_thresholds( def concat( - this: Union[Array[numpy.str_], ArrayLike[str]], - that: Union[Array[numpy.str_], ArrayLike[str]], -) -> Array[numpy.str_]: + this: Union[t.Array[numpy.str_], t.ArrayLike[str]], + that: Union[t.Array[numpy.str_], t.ArrayLike[str]], +) -> t.Array[numpy.str_]: """Concatenates the values of two arrays. Args: @@ -89,9 +89,9 @@ def concat( def switch( - conditions: Array[numpy.float_], + conditions: t.Array[numpy.float_], value_by_condition: Mapping[float, float], -) -> Array[numpy.float_]: +) -> t.Array[numpy.float_]: """Mimicks a switch statement. Given an array of conditions, returns an array of the same size, diff --git a/openfisca_core/commons/misc.py b/openfisca_core/commons/misc.py index ff1c2e386..342bbbe5f 100644 --- a/openfisca_core/commons/misc.py +++ b/openfisca_core/commons/misc.py @@ -1,9 +1,9 @@ from typing import Optional, TypeVar -from openfisca_core.types import Array - import numpy +from openfisca_core import types as t + T = TypeVar("T") @@ -45,7 +45,7 @@ def empty_clone(original: T) -> T: return new -def stringify_array(array: Optional[Array[numpy.generic]]) -> str: +def stringify_array(array: Optional[t.Array[numpy.generic]]) -> str: """Generates a clean string representation of a numpy array. Args: diff --git a/openfisca_core/commons/py.typed b/openfisca_core/commons/py.typed new file mode 100644 index 000000000..e69de29bb From 5aaca8690e3034a3083c4dea0b11fa7a066c1930 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 17 Sep 2024 01:13:34 +0200 Subject: [PATCH 7/7] chore(version): bump --- CHANGELOG.md | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75fdd9ed3..4b0f3e63a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +### 41.5.4 [#1219](https://github.com/openfisca/openfisca-core/pull/1219) + +#### Technical changes + +- Fix doc & type definitions in the commons module + ### 41.5.3 [#1218](https://github.com/openfisca/openfisca-core/pull/1218) #### Technical changes diff --git a/setup.py b/setup.py index e822e138b..92971f40b 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setup( name="OpenFisca-Core", - version="41.5.3", + version="41.5.4", author="OpenFisca Team", author_email="contact@openfisca.org", classifiers=[