Runtime and structural changes to compile key modules to C with mypyc

The bulk of the mypyc port: replace custom metaclasses with __init_subclass__
plus explicitly written-out resource members, migrate from the home-grown
cached_property to functools.cached_property, add mypyc_attr directives,
native-class type-narrowing asserts/casts, split PackageOrderList out into
package_order_list.py, and assorted runtime fixes.

Pure type-annotation changes are split out into separate commits (the
package-resource annotations are folded into "Add generics for package
resources"; the rest into "Add type annotations to assorted modules").

Author: chadrik

.gitignore

@@ -23,3 +23,5 @@ docs/source/commands/
 __tests_pkg_repo/
 .idea/
 venv/
+/out*
+/build*

mypyc-custom.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+#pip install ../mypy
+pip uninstall -y rez
+pip install --no-build-isolation -e .
+python -m rez.cli._main selftest --debug

mypyc-test.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+builddir=./build
+
+rm -rf $builddir
+_REZ_NO_KILLPG=1 python ./install.py -v $builddir
+
+
+#$builddir/bin/rez/rez-selftest
+
+#_REZ_NO_KILLPG=1 ./build/bin/python -c "import rez.utils.resources; print(rez.utils.resources.__file__); "
+#_REZ_NO_KILLPG=1 ./build/bin/python -c "import rez.solver; print(rez.solver.__file__)"
+#_REZ_NO_KILLPG=1 ./build/bin/python -c "import rez.resolver; print(rez.resolver.__file__)"
+#_REZ_NO_KILLPG=1 ./build/bin/python -c "import rez.packages; print(rez.packages.__file__)"
+# ./build/bin/python -c "import rez.version._version; print(rez.version._version.__file__)"
+
+./build/bin/rez/rez-selftest
+
+# ./build/bin/rez/rez-benchmark --out ./out-solver-resolver
+
+# PYTHONPATH=src python -c "import rez.utils.data_utils;rez.utils.data_utils.write_all_dynamic_members()"
+
+# python -m rez.cli._main selftest

pyproject.toml

@@ -1,30 +1,82 @@
 [build-system]
-requires = ["setuptools"]
+requires = ["setuptools", "mypy[mypyc]"]
 build-backend = "setuptools.build_meta"
 
 [tool.mypy]
 python_version = "3.8"
 files = ["src/rez/", "src/rezplugins/"]
 exclude = [
     '.*/rez/data/.*',
-    '.*/rez/vendor/.*',
     '.*/rez/tests/.*',
     '.*/rez/utils/lint_helper.py',
 ]
 disable_error_code = ["var-annotated", "import-not-found"]
 check_untyped_defs = true
 # allow this for now:
-allow_redefinition = true
+#allow_redefinition = true
 follow_imports = "silent"
+enable_incomplete_feature = ["InlineTypedDict"]
+
+[[tool.mypy.overrides]]
+module = "rez.vendor.*"
+ignore_errors = true
+
+[[tool.mypy.overrides]]
+module = "rez.vendor.schema.schema"
+ignore_errors = false
+
+[[tool.mypy.overrides]]
+module = "rez.utils.formatting"
+disallow_untyped_defs = true
 
 [[tool.mypy.overrides]]
 module = 'rez.utils.lint_helper'
 follow_imports = "skip"
 
 [[tool.mypy.overrides]]
-module = "rez.version._version"
+module = "rez.packages"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.package_filter"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.package_order"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.package_repository"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.package_resources"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.solver"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.resolved_context"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.resolver"
 disallow_untyped_defs = true
 
 [[tool.mypy.overrides]]
-module = "rez.version._requirement"
+module = "rez.version.*"
 disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.utils.resources"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.utils.scope"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rezplugins.package_repository.filesystem"
+disallow_untyped_defs = true

setup.py

@@ -43,6 +43,36 @@ def find_files(pattern, path=None, root="rez"):
     long_description = f.read()
 
 
+if os.environ.get("REZ_MYPYC", "1") != "0":
+    from mypyc.build import mypycify
+    ext_modules = mypycify(
+        [
+            # "src/rez/package_filter.py",  # errors at runtime calling cached_property.uncache: "attribute 'cost' of 'PackageFilter' objects is not writable"
+            "src/rez/solver.py",
+            # "src/rez/resolved_context.py",  # error in copy().  need to rework from_dict()
+            "src/rez/resolver.py",
+            "src/rez/package_order.py",  # split out PackageOrderList from this module due to inheriting from list
+            "src/rez/package_repository.py",
+            "src/rez/package_resources.py",  # requires fixed version of mypyc
+            "src/rez/version/__init__.py",
+            "src/rez/version/_util.py",
+            "src/rez/version/_requirement.py",
+            "src/rez/version/_version.py",
+            "src/rez/utils/formatting.py",  # includes a mixin, which should not be compiled
+            # "src/rez/utils/memcached.py",
+            "src/rez/utils/resources.py",
+            # "src/rez/utils/scope.py",  # objects directly access/modify __dict__
+            # "src/rez/vendor/schema/schema.py",
+            "src/rezplugins/package_repository/filesystem.py",  # ConfigurationError at runtime with FileSystemPackageRepository plugin
+        ],
+        # only_compile_paths=["src/rez/solver.py"]
+        opt_level="3",
+        multi_file=True
+    )
+else:
+    ext_modules = []
+
+
 setup(
     name="rez",
     version=_rez_version,
@@ -68,7 +98,8 @@ def find_files(pattern, path=None, root="rez"):
     packages=find_packages('src', exclude=["build_utils",
                                            "build_utils.*",
                                            "tests"]),
-    install_requires=[],
+    install_requires=["mypy_extensions"],
+    ext_modules=ext_modules,
     package_data={
         'rez':
             ['utils/logging.conf'] +

src/rez/cli/env.py

@@ -197,7 +197,7 @@ def command(opts, parser, extra_arg_groups=None) -> None:
         if opts.no_filters:
             package_filter = PackageFilterList()
         else:
-            package_filter = PackageFilterList.singleton.copy()
+            package_filter = PackageFilterList.singleton().copy()
 
         for rule_str in (opts.exclude or []):
             rule = Rule.parse_rule(rule_str)

src/rez/config.py

@@ -6,8 +6,9 @@
 
 from rez import __version__
 from rez.utils.data_utils import AttrDictWrapper, RO_AttrDictWrapper, \
-    convert_dicts, cached_property, cached_class_property, LazyAttributeMeta, \
+    convert_dicts, cached_class_property, LazyAttributeMeta, \
     deep_update, ModifyList, DelayLoad
+from functools import cached_property
 from rez.utils.formatting import expandvars, expanduser
 from rez.utils.logging_ import get_debug_printer
 from rez.utils.scope import scoped_format

src/rez/package_copy.py

@@ -241,6 +241,8 @@ def finalize():
                 variant_resource=src_variant.resource,
                 overrides=overrides_
             )
+            # not a dry run, so a resource must have been returned
+            assert dest_variant_resource is not None
 
             dest_variant = Variant(dest_variant_resource)
 

src/rez/package_filter.py

@@ -7,16 +7,54 @@
 from rez.packages import iter_packages, Package
 from rez.exceptions import ConfigurationError
 from rez.config import config
-from rez.utils.data_utils import cached_property, cached_class_property
+from functools import cached_property
 from rez.version import VersionedObject, VersionRange, Requirement
 from hashlib import sha1
-from typing import Any, Iterator, Pattern, TYPE_CHECKING, ClassVar
+from typing import cast, Any, Iterator, Pattern, TYPE_CHECKING, ClassVar
 import fnmatch
 import re
 
 if TYPE_CHECKING:
     from typing import Self
 
+import functools
+from typing import Callable, Generic, TypeVar
+T = TypeVar("T")
+TypeT = TypeVar("TypeT", bound=type)
+
+class cached_class_property(Generic[T]):
+    """Simple class property caching descriptor.
+
+    Example:
+
+        >>> class Foo(object):
+        >>>     @cached_class_property
+        >>>     def bah(cls):
+        >>>         print('bah')
+        >>>         return 1
+        >>>
+        >>> Foo.bah
+        bah
+        1
+        >>> Foo.bah
+        1
+    """
+    def __init__(self, func: Callable[[], T], name: str | None = None) -> None:
+        self.func = func
+        # Make sure that Sphinx autodoc can follow and get the docstring from our wrapped function.
+        # TODO: Doesn't work...
+        functools.update_wrapper(self, func)  # type: ignore[arg-type]
+
+    def __get__(self, instance: Any, owner: Any | None = None) -> T:
+        assert owner
+        name = "_class_property_" + self.func.__name__
+        result: T | type[KeyError] = getattr(owner, name, KeyError)
+
+        if result is KeyError:
+            result = self.func()
+            setattr(owner, name, result)
+        return cast(T, result)
+
 
 class PackageFilterBase(object):
     """Base class for package filters."""
@@ -58,7 +96,6 @@ def to_pod(self) -> Any:
         """Convert to POD type, suitable for storing in an rxt file.
 
         Return type depends on subclass implementation
-            dict[str, list[str]]:
         """
         raise NotImplementedError
 
@@ -220,7 +257,8 @@ def _add_rule(self, rules_dict: dict[str | None, list[Rule]], rule: Rule) -> Non
         family = rule.family()
         rules_ = rules_dict.get(family, [])
         rules_dict[family] = sorted(rules_ + [rule], key=lambda x: x.cost())
-        cached_property.uncache(self, "cost")  # type: ignore[attr-defined]
+        # invalidate the functools.cached_property cache entry
+        self.__dict__.pop("cost", None)
 
     def __str__(self) -> str:
         def sortkey(rule_items: tuple[str | None, list[Rule]]) -> tuple[str, list[Rule]]:
@@ -233,6 +271,15 @@ def sortkey(rule_items: tuple[str | None, list[Rule]]) -> tuple[str, list[Rule]]
                     sorted(self._includes.items(), key=sortkey)))
 
 
+def _singleton() -> PackageFilterList:
+    """Filter list as configured by :data:`package_filter`.
+
+    Returns:
+        PackageFilterList:
+    """
+    return PackageFilterList.from_pod(config.package_filter)
+
+
 class PackageFilterList(PackageFilterBase):
     """A list of package filters.
 
@@ -318,7 +365,7 @@ def __str__(self) -> str:
         filters = sorted(self.filters, key=lambda x: (x.cost, str(x)))
         return str(tuple(filters))
 
-    @cached_class_property
+    @classmethod
     def singleton(cls) -> PackageFilterList:
         """Filter list as configured by :data:`package_filter`.
 
@@ -327,6 +374,8 @@ def singleton(cls) -> PackageFilterList:
         """
         return cls.from_pod(config.package_filter)
 
+    # singleton = cached_class_property(_singleton)
+
 
 #: filter that does not exclude any packages
 no_filter = PackageFilterList()