Re-extend NumPy compatibility to v1.20

Merge pull request #1010 from openfisca/numpy-typing

Author: sandcha

.circleci/config.yml

@@ -33,6 +33,10 @@ jobs:
           name: Run Core tests
           command: env PYTEST_ADDOPTS="--exitfirst" make test
 
+      - run:
+          name: Check NumPy typing against latest 3 minor versions
+          command: for i in {1..3}; do VERSION=$(.circleci/get-numpy-version.py prev) && pip install numpy==$VERSION && make check-types; done
+
       - persist_to_workspace:
           root: .
           paths:

.circleci/get-numpy-version.py

@@ -0,0 +1,38 @@
+#! /usr/bin/env python
+
+from __future__ import annotations
+
+import os
+import sys
+import typing
+from packaging import version
+from typing import NoReturn, Union
+
+import numpy
+
+if typing.TYPE_CHECKING:
+    from packaging.version import LegacyVersion, Version
+
+
+def prev() -> NoReturn:
+    release = _installed().release
+
+    if release is None:
+        sys.exit(os.EX_DATAERR)
+
+    major, minor, _ = release
+
+    if minor == 0:
+        sys.exit(os.EX_DATAERR)
+
+    minor -= 1
+    print(f"{major}.{minor}.0")  # noqa: T001
+    sys.exit(os.EX_OK)
+
+
+def _installed() -> Union[LegacyVersion, Version]:
+    return version.parse(numpy.__version__)
+
+
+if __name__ == "__main__":
+    globals()[sys.argv[1]]()

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 35.4.0 [#1010](https://github.com/openfisca/openfisca-core/pull/1010)
+
+#### Technical changes
+
+- Update dependencies (_as in 35.3.7_).
+  - Extend NumPy compatibility to v1.20 to support M1 processors.
+
+- Make NumPy's type-checking compatible with 1.17.0+
+  - NumPy introduced their `typing` module since 1.20.0
+  - Previous type hints relying on `annotations` will henceforward no longer work
+  - This changes ensure that type hints are always legal for the last four minor NumPy versions
+
 ### 35.3.8 [#1014](https://github.com/openfisca/openfisca-core/pull/1014)
 
 #### Bug fix

README.md

@@ -17,6 +17,8 @@ This package contains the core features of OpenFisca, which are meant to be used
 
 OpenFisca runs on Python 3.7. More recent versions should work, but are not tested.
 
+OpenFisca also relies strongly on NumPy. Last four minor versions should work, but only latest/stable is tested.
+
 ## Installation
 
 If you're developing your own country package, you don't need to explicitly install OpenFisca-Core. It just needs to appear [in your package dependencies](https://github.com/openfisca/openfisca-france/blob/18.2.1/setup.py#L53).
@@ -49,6 +51,18 @@ To run a single test:
 pytest tests/core/test_parameters.py -k test_parameter_for_period
 ```
 
+## Types
+
+This repository relies on MyPy for optional dynamic & static type checking.
+
+As NumPy introduced the `typing` module in 1.20.0, to ensure type hints do not break the code at runtime, we run the checker against the last four minor NumPy versions.
+
+Type checking is already run with `make test`. To run the type checker alone:
+
+```sh
+make check-types
+```
+
 ## Style
 
 This repository adheres to a [certain coding style](STYLEGUIDE.md), and we invite you to follow it for your contributions to be integrated promptly.

STYLEGUIDE.md

@@ -42,10 +42,10 @@ Whenever possible we should expect:
 # Yes
 
 import copy
+import typing
 from typing import List
 
 import numpy
-from numpy.typing import ArrayLike
 
 from openfisca_country_template import entities
 
@@ -54,6 +54,10 @@ from openfisca_core.variables import Variable
 
 from . import Something
 
+if typing.TYPE_CHECKING:
+    from numpy.typing import ArrayLike
+
+
 def do(this: List) -> ArrayLike:
     that = copy.deepcopy(this)
     array = numpy.ndarray(that)

openfisca_core/indexed_enums/enum.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import enum
-import typing
+from typing import Union
 
 import numpy
 
@@ -10,37 +10,41 @@
 
 class Enum(enum.Enum):
     """
-    Enum based on `enum34 <https://pypi.python.org/pypi/enum34/>`_, whose items have an
-    index.
+    Enum based on `enum34 <https://pypi.python.org/pypi/enum34/>`_, whose items
+    have an index.
     """
 
     # Tweak enums to add an index attribute to each enum item
     def __init__(self, name: str) -> None:
-        # When the enum item is initialized, self._member_names_ contains the names of
-        # the previously initialized items, so its length is the index of this item.
+        # When the enum item is initialized, self._member_names_ contains the
+        # names of the previously initialized items, so its length is the index
+        # of this item.
         self.index = len(self._member_names_)
 
     # Bypass the slow Enum.__eq__
     __eq__ = object.__eq__
 
-    # In Python 3, __hash__ must be defined if __eq__ is defined to stay hashable.
+    # In Python 3, __hash__ must be defined if __eq__ is defined to stay
+    # hashable.
     __hash__ = object.__hash__
 
     @classmethod
     def encode(
             cls,
-            array: typing.Union[
+            array: Union[
                 EnumArray,
-                numpy.ndarray[int],
-                numpy.ndarray[str],
-                numpy.ndarray[Enum],
+                numpy.int_,
+                numpy.float_,
+                numpy.object_,
                 ],
             ) -> EnumArray:
         """
-        Encode a string numpy array, an enum item numpy array, or an int numpy array
-        into an :any:`EnumArray`. See :any:`EnumArray.decode` for decoding.
+        Encode a string numpy array, an enum item numpy array, or an int numpy
+        array into an :any:`EnumArray`. See :any:`EnumArray.decode` for
+        decoding.
 
-        :param ndarray array: Array of string identifiers, or of enum items, to encode.
+        :param ndarray array: Array of string identifiers, or of enum items, to
+                              encode.
 
         :returns: An :any:`EnumArray` encoding the input array values.
         :rtype: :any:`EnumArray`
@@ -59,24 +63,31 @@ def encode(
         >>> encoded_array[0]
         2  # Encoded value
         """
-        if type(array) is EnumArray:
+        if isinstance(array, EnumArray):
             return array
 
-        if array.dtype.kind in {'U', 'S'}:  # String array
+        # String array
+        if isinstance(array, numpy.ndarray) and \
+                array.dtype.kind in {'U', 'S'}:
             array = numpy.select(
                 [array == item.name for item in cls],
                 [item.index for item in cls],
                 ).astype(config.ENUM_ARRAY_DTYPE)
 
-        elif array.dtype.kind == 'O':  # Enum items arrays
+        # Enum items arrays
+        elif isinstance(array, numpy.ndarray) and \
+                array.dtype.kind == 'O':
             # Ensure we are comparing the comparable. The problem this fixes:
             # On entering this method "cls" will generally come from
-            # variable.possible_values, while the array values may come from directly
-            # importing a module containing an Enum class. However, variables (and
-            # hence their possible_values) are loaded by a call to load_module, which
-            # gives them a different identity from the ones imported in the usual way.
-            # So, instead of relying on the "cls" passed in, we use only its name to
-            # check that the values in the array, if non-empty, are of the right type.
+            # variable.possible_values, while the array values may come from
+            # directly importing a module containing an Enum class. However,
+            # variables (and hence their possible_values) are loaded by a call
+            # to load_module, which gives them a different identity from the
+            # ones imported in the usual way.
+            #
+            # So, instead of relying on the "cls" passed in, we use only its
+            # name to check that the values in the array, if non-empty, are of
+            # the right type.
             if len(array) > 0 and cls.__name__ is array[0].__class__.__name__:
                 cls = array[0].__class__
 

openfisca_core/indexed_enums/enum_array.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import typing
+from typing import Any, NoReturn, Optional, Type
 
 import numpy
 
@@ -20,37 +21,37 @@ class EnumArray(numpy.ndarray):
     # https://docs.scipy.org/doc/numpy-1.13.0/user/basics.subclassing.html#slightly-more-realistic-example-attribute-added-to-existing-array.
     def __new__(
             cls,
-            input_array: numpy.ndarray[int],
-            possible_values: typing.Optional[typing.Type[Enum]] = None,
+            input_array: numpy.int_,
+            possible_values: Optional[Type[Enum]] = None,
             ) -> EnumArray:
         obj = numpy.asarray(input_array).view(cls)
         obj.possible_values = possible_values
         return obj
 
     # See previous comment
-    def __array_finalize__(self, obj: typing.Optional[numpy.ndarray[int]]) -> None:
+    def __array_finalize__(self, obj: Optional[numpy.int_]) -> None:
         if obj is None:
             return
 
         self.possible_values = getattr(obj, "possible_values", None)
 
-    def __eq__(self, other: typing.Any) -> bool:
-        # When comparing to an item of self.possible_values, use the item index to
-        # speed up the comparison.
+    def __eq__(self, other: Any) -> bool:
+        # When comparing to an item of self.possible_values, use the item index
+        # to speed up the comparison.
         if other.__class__.__name__ is self.possible_values.__name__:
             # Use view(ndarray) so that the result is a classic ndarray, not an
             # EnumArray.
             return self.view(numpy.ndarray) == other.index
 
         return self.view(numpy.ndarray) == other
 
-    def __ne__(self, other: typing.Any) -> bool:
+    def __ne__(self, other: Any) -> bool:
         return numpy.logical_not(self == other)
 
-    def _forbidden_operation(self, other: typing.Any) -> typing.NoReturn:
+    def _forbidden_operation(self, other: Any) -> NoReturn:
         raise TypeError(
-            "Forbidden operation. The only operations allowed on EnumArrays are "
-            "'==' and '!='.",
+            "Forbidden operation. The only operations allowed on EnumArrays "
+            "are '==' and '!='.",
             )
 
     __add__ = _forbidden_operation
@@ -62,7 +63,7 @@ def _forbidden_operation(self, other: typing.Any) -> typing.NoReturn:
     __and__ = _forbidden_operation
     __or__ = _forbidden_operation
 
-    def decode(self) -> numpy.ndarray[Enum]:
+    def decode(self) -> numpy.object_:
         """
         Return the array of enum items corresponding to self.
 
@@ -72,14 +73,16 @@ def decode(self) -> numpy.ndarray[Enum]:
         >>> enum_array[0]
         >>> 2  # Encoded value
         >>> enum_array.decode()[0]
-        <HousingOccupancyStatus.free_lodger: 'Free lodger'>  # Decoded value : enum item
+        <HousingOccupancyStatus.free_lodger: 'Free lodger'>
+
+        Decoded value: enum item
         """
         return numpy.select(
             [self == item.index for item in self.possible_values],
             list(self.possible_values),
             )
 
-    def decode_to_str(self) -> numpy.ndarray[str]:
+    def decode_to_str(self) -> numpy.str_:
         """
         Return the array of string identifiers corresponding to self.
 

openfisca_core/taxscales/abstract_rate_tax_scale.py

@@ -3,28 +3,36 @@
 import typing
 import warnings
 
-import numpy
-
 from openfisca_core.taxscales import RateTaxScaleLike
 
+if typing.TYPE_CHECKING:
+    import numpy
+
+    NumericalArray = typing.Union[numpy.int_, numpy.float_]
+
 
 class AbstractRateTaxScale(RateTaxScaleLike):
     """
-    Base class for various types of rate-based tax scales: marginal rate, linear
-    average rate...
+    Base class for various types of rate-based tax scales: marginal rate,
+    linear average rate...
     """
 
-    def __init__(self, name: typing.Optional[str] = None, option = None, unit = None) -> None:
+    def __init__(
+            self, name: typing.Optional[str] = None,
+            option: typing.Any = None,
+            unit: typing.Any = None,
+            ) -> None:
         message = [
-            "The 'AbstractRateTaxScale' class has been deprecated since version",
-            "34.7.0, and will be removed in the future.",
+            "The 'AbstractRateTaxScale' class has been deprecated since",
+            "version 34.7.0, and will be removed in the future.",
             ]
+
         warnings.warn(" ".join(message), DeprecationWarning)
-        super(AbstractRateTaxScale, self).__init__(name, option, unit)
+        super().__init__(name, option, unit)
 
     def calc(
             self,
-            tax_base: typing.Union[numpy.ndarray[int], numpy.ndarray[float]],
+            tax_base: NumericalArray,
             right: bool,
             ) -> typing.NoReturn:
         raise NotImplementedError(