Add the option to compile key modules to C using mypyc

.github/workflows/benchmark.yaml

@@ -21,7 +21,8 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - '3.7'
+          - '3.10'
+        mypyc: [false, true]
 
     steps:
       - name: Checkout
@@ -33,6 +34,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install Rez
+        env:
+          REZ_MYPYC: ${{ matrix.mypyc && '1' || '0' }}
         run: |
           mkdir ./installdir
 
@@ -51,7 +54,7 @@ jobs:
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: "benchmark-result-${{ matrix.python-version }}"
+          name: "benchmark-result-${{ matrix.python-version }}-mypyc-${{ matrix.mypyc }}"
           path: ./out
 
   store_benchmark_result:
@@ -62,15 +65,15 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - '3.7'
+          - '3.10'
 
       # so we don't have jobs trying to push to git at the same time
       max-parallel: 1
 
     steps:
       - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: "benchmark-result-${{ matrix.python-version }}"
+          name: "benchmark-result-${{ matrix.python-version }}-mypyc-false"
           path: .
 
       - name: Checkout (release)

.github/workflows/installation.yaml

@@ -27,14 +27,22 @@ permissions:
 
 jobs:
   main:
-    name: ${{ matrix.os }} - ${{ matrix.python-version }} - ${{ matrix.method }}
+    name: ${{ matrix.os }} - ${{ matrix.python-version }} - ${{ matrix.method }} - mypyc=${{ matrix.mypyc }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
         python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
         method: ['install' ,'pip']
+        mypyc: [false, true]
+
+        exclude:
+        # mypyc requires Python >= 3.10 (mypy does not support older versions)
+        - python-version: '3.8'
+          mypyc: true
+        - python-version: '3.9'
+          mypyc: true
 
         include:
         # ubuntu
@@ -76,6 +84,7 @@ jobs:
     - name: Install
       env:
         MATRIX_PYTHON_VERSION: ${{ matrix.python-version }}
+        REZ_MYPYC: ${{ matrix.mypyc && '1' || '0' }}
       run: |
         ${{ matrix.REZ_INSTALL_COMMAND }}
 

.github/workflows/mypy.yaml

@@ -38,7 +38,7 @@ jobs:
 
       - name: Install Dependencies
         run: |
-          pip install mypy==1.14.1 mypy_baseline==0.7.1 types-setuptools
+          pip install mypy==2.1.0 mypy_baseline==0.7.1 types-setuptools
 
       - name: Run mypy
         run: >-

.github/workflows/tests.yaml

@@ -39,6 +39,15 @@ jobs:
       matrix:
         os: ['macos-latest', 'ubuntu-latest', 'windows-latest']
         python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+        mypyc: [false, true]
+
+        exclude:
+        # mypyc requires Python >= 3.10 (mypy does not support older versions)
+        - python-version: '3.8'
+          mypyc: true
+        - python-version: '3.9'
+          mypyc: true
+
         include:
         - os: macos-latest
           shells: 'sh,csh,bash,tcsh,zsh,pwsh'
@@ -73,6 +82,8 @@ jobs:
         sudo apt-get install -y csh tcsh zsh
 
     - name: Install rez
+      env:
+        REZ_MYPYC: ${{ matrix.mypyc && '1' || '0' }}
       run: python ./install.py ./installdir
 
     - name: Install test system dependencies (macOS)

.gitignore

@@ -23,3 +23,5 @@ docs/source/commands/
 __tests_pkg_repo/
 .idea/
 venv/
+/out*
+/build*

mypyc-custom.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+#pip install ../mypy
+pip uninstall -y rez
+pip install --no-build-isolation -e .
+python -m rez.cli._main selftest --debug

mypyc-test.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+builddir=./build
+
+rm -rf $builddir
+_REZ_NO_KILLPG=1 python ./install.py -v $builddir
+
+
+#$builddir/bin/rez/rez-selftest
+
+#_REZ_NO_KILLPG=1 ./build/bin/python -c "import rez.utils.resources; print(rez.utils.resources.__file__); "
+#_REZ_NO_KILLPG=1 ./build/bin/python -c "import rez.solver; print(rez.solver.__file__)"
+#_REZ_NO_KILLPG=1 ./build/bin/python -c "import rez.resolver; print(rez.resolver.__file__)"
+#_REZ_NO_KILLPG=1 ./build/bin/python -c "import rez.packages; print(rez.packages.__file__)"
+# ./build/bin/python -c "import rez.version._version; print(rez.version._version.__file__)"
+
+./build/bin/rez/rez-selftest
+
+# ./build/bin/rez/rez-benchmark --out ./out-solver-resolver
+
+# PYTHONPATH=src python -c "import rez.utils.data_utils;rez.utils.data_utils.write_all_dynamic_members()"
+
+# python -m rez.cli._main selftest

pyproject.toml

@@ -1,30 +1,101 @@
 [build-system]
-requires = ["setuptools"]
+# Building rez with mypyc (the default; set REZ_MYPYC=0 to build pure-python)
+# requires our fork of mypy, which carries fixes needed to compile rez. Those
+# fixes are pending upstreaming; until they land, pin the fork so that an
+# isolated build (pip install ., install.py, etc.) picks up the right mypy.
+#
+# Local testing against a working copy of the mypy fork:
+#   The pin below points at the *pushed* fork branch on GitHub, so an isolated
+#   build always fetches it from the network and ignores any local mypy changes.
+#   To build against a local checkout instead, either:
+#     (a) recommended - install the fork editable in your venv and build with
+#         --no-build-isolation, which uses that installed mypy and ignores the
+#         requirement below entirely (this is what ./mypyc-custom.sh does):
+#             pip install -e /path/to/mypy
+#             pip install --no-build-isolation -e .
+#     (b) for an isolated build, temporarily replace the requirement below with
+#         a local path (revert before committing):
+#             "mypy[mypyc] @ file:///path/to/mypy"
+requires = ["setuptools", "mypy[mypyc] @ git+https://github.com/chadrik/mypy@rez-mypyc-fixes ; python_version >= '3.10'"]
 build-backend = "setuptools.build_meta"
 
 [tool.mypy]
-python_version = "3.8"
+# mypy 2.1 requires the type-check target to be 3.10+, so we can no longer
+# target rez's lowest supported runtime (3.8). 3.10 is the closest mypy allows.
+python_version = "3.10"
 files = ["src/rez/", "src/rezplugins/"]
 exclude = [
     '.*/rez/data/.*',
-    '.*/rez/vendor/.*',
     '.*/rez/tests/.*',
     '.*/rez/utils/lint_helper.py',
 ]
 disable_error_code = ["var-annotated", "import-not-found"]
 check_untyped_defs = true
 # allow this for now:
-allow_redefinition = true
+#allow_redefinition = true
 follow_imports = "silent"
+enable_incomplete_feature = ["InlineTypedDict"]
+
+[[tool.mypy.overrides]]
+module = "rez.vendor.*"
+ignore_errors = true
+
+[[tool.mypy.overrides]]
+module = "rez.vendor.schema.schema"
+ignore_errors = false
+
+[[tool.mypy.overrides]]
+module = "rez.utils.formatting"
+disallow_untyped_defs = true
 
 [[tool.mypy.overrides]]
 module = 'rez.utils.lint_helper'
 follow_imports = "skip"
 
 [[tool.mypy.overrides]]
-module = "rez.version._version"
+module = "rez.packages"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.package_filter"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.package_order"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.package_repository"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.package_resources"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.solver"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.resolved_context"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.resolver"
 disallow_untyped_defs = true
 
 [[tool.mypy.overrides]]
-module = "rez.version._requirement"
+module = "rez.version.*"
 disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.utils.resources"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rez.utils.scope"
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "rezplugins.package_repository.filesystem"
+disallow_untyped_defs = true

setup.py

@@ -43,6 +43,49 @@ def find_files(pattern, path=None, root="rez"):
     long_description = f.read()
 
 
+# Compile with mypyc only when explicitly requested via REZ_MYPYC=1. The
+# default (and REZ_MYPYC=0) is a pure python build, regardless of python
+# version. Requesting a mypyc build on a python that mypy does not support
+# (< 3.10) is a hard error rather than a silent fallback.
+use_mypyc = os.environ.get("REZ_MYPYC") == "1"
+
+if use_mypyc and sys.version_info < (3, 10):
+    raise RuntimeError(
+        "Building rez with mypyc requires Python 3.10 or newer (mypy does "
+        "not support older versions). Use Python >= 3.10, or set "
+        "REZ_MYPYC=0 to build pure python."
+    )
+
+if use_mypyc:
+    from mypyc.build import mypycify
+    ext_modules = mypycify(
+        [
+            # "src/rez/package_filter.py",  # errors at runtime calling cached_property.uncache: "attribute 'cost' of 'PackageFilter' objects is not writable"
+            "src/rez/solver.py",
+            # "src/rez/resolved_context.py",  # error in copy().  need to rework from_dict()
+            "src/rez/resolver.py",
+            "src/rez/package_order.py",  # split out PackageOrderList from this module due to inheriting from list
+            "src/rez/package_repository.py",
+            "src/rez/package_resources.py",  # requires fixed version of mypyc
+            "src/rez/version/__init__.py",
+            "src/rez/version/_util.py",
+            "src/rez/version/_requirement.py",
+            "src/rez/version/_version.py",
+            "src/rez/utils/formatting.py",  # includes a mixin, which should not be compiled
+            # "src/rez/utils/memcached.py",
+            "src/rez/utils/resources.py",
+            # "src/rez/utils/scope.py",  # objects directly access/modify __dict__
+            # "src/rez/vendor/schema/schema.py",
+            "src/rezplugins/package_repository/filesystem.py",  # ConfigurationError at runtime with FileSystemPackageRepository plugin
+        ],
+        # only_compile_paths=["src/rez/solver.py"]
+        opt_level="3",
+        multi_file=True
+    )
+else:
+    ext_modules = []
+
+
 setup(
     name="rez",
     version=_rez_version,
@@ -68,7 +111,8 @@ def find_files(pattern, path=None, root="rez"):
     packages=find_packages('src', exclude=["build_utils",
                                            "build_utils.*",
                                            "tests"]),
-    install_requires=[],
+    install_requires=["mypy_extensions"],
+    ext_modules=ext_modules,
     package_data={
         'rez':
             ['utils/logging.conf'] +

src/rez/__init__.py

@@ -58,10 +58,7 @@ def callback(sig, frame) -> None:
             txt = ''.join(traceback.format_stack(frame))
             print()
             print(txt)
-    else:
-        callback = None
 
-    if callback:
         signal.signal(signal.SIGUSR1, callback)  # Register handler
 
 

src/rez/build_process.py

@@ -404,10 +404,15 @@ def get_changelog(self) -> str | None:
             previous_revision = None
 
         changelog = None
-        with self.repo_operation():
-            changelog = self.vcs.get_changelog(
-                previous_revision,
-                max_revisions=config.max_package_changelog_revisions)
+        # self.vcs is Optional; with no vcs there is no changelog to compute.
+        # Both callers already treat a None changelog as "no changelog" (one
+        # only reaches here after a vcs-is-None early return, the other catches
+        # and nulls it), so skipping the lookup when there is no vcs is safe.
+        if self.vcs:
+            with self.repo_operation():
+                changelog = self.vcs.get_changelog(
+                    previous_revision,
+                    max_revisions=config.max_package_changelog_revisions)
 
         return changelog
 

src/rez/cli/_complete_util.py

@@ -96,9 +96,10 @@ def __call__(self, prefix, **kwargs):
             return []
 
         matching_names = []
-        names = (x for x in names if x.startswith(fileprefix))
 
         for name in names:
+            if not name.startswith(fileprefix):
+                continue
             filepath = os.path.join(path, name)
             if os.path.isfile(filepath):
                 if not self.files:

src/rez/cli/benchmark.py

@@ -49,6 +49,8 @@ def load_packages() -> None:
     """
     from rez.packages import iter_package_families
 
+    assert pkg_repo_dir is not None
+
     print("Warming package cache...")
     fams = list(iter_package_families(paths=[pkg_repo_dir]))
 

src/rez/cli/bind.py

@@ -54,9 +54,9 @@ def command(opts, parser, extra_arg_groups=None) -> None:
 
     if opts.list:
         d = get_bind_modules()
-        rows = [["PACKAGE", "BIND MODULE"],
-                ["-------", "-----------"]]
-        rows += sorted(d.items())
+        rows = [("PACKAGE", "BIND MODULE"),
+                ("-------", "-----------")]
+        rows += sorted((k, v) for k, v in d.items())
         print('\n'.join(columnise(rows)))
         return
 

src/rez/cli/interpret.py

@@ -56,7 +56,6 @@ def command(opts, parser, extra_arg_groups=None) -> None:
         with open(filename) as f:
             code = f.read()
 
-    interp = None
     if opts.format is None:
         interp = create_shell()
     elif opts.format in ('dict', 'table'):