Merge branch 'master' into neopexrl-patch-1

Author: neopexrl

.github/workflows/test.yml

@@ -79,12 +79,18 @@ jobs:
         #   # allow_failure: true
         #   test_mypyc: true
 
-        - name: mypyc runtime tests with py39-macos
-          python: '3.9.21'
-          # TODO: macos-13 is the last one to support Python 3.9, change it to macos-latest when updating the Python version
-          os: macos-13
+        - name: mypyc runtime tests with py313-macos
+          python: '3.13'
+          os: macos-latest
           toxenv: py
           tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py"
+
+        - name: mypyc runtime tests with py310-ubuntu
+          python: '3.10'
+          os: ubuntu-latest
+          toxenv: py
+          tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py"
+
         # This is broken. See
         # - https://github.com/python/mypy/issues/17819
         # - https://github.com/python/mypy/pull/17822

.pre-commit-config.yaml

@@ -6,14 +6,14 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 25.1.0
+    rev: 25.9.0
     hooks:
       - id: black
         exclude: '^(test-data/)'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.4
+    rev: v0.14.3
     hooks:
-      - id: ruff
+      - id: ruff-check
         args: [--exit-non-zero-on-fix]
   - repo: https://github.com/python-jsonschema/check-jsonschema
     rev: 0.32.1

docs/source/command_line.rst

@@ -1159,7 +1159,7 @@ format into the specified directory.
 Enabling incomplete/experimental features
 *****************************************
 
-.. option:: --enable-incomplete-feature {PreciseTupleTypes, InlineTypedDict}
+.. option:: --enable-incomplete-feature {PreciseTupleTypes,InlineTypedDict,TypeForm}
 
     Some features may require several mypy releases to implement, for example
     due to their complexity, potential for backwards incompatibility, or
@@ -1211,8 +1211,11 @@ List of currently incomplete/experimental features:
 
   .. code-block:: python
 
-     def test_values() -> {"int": int, "str": str}:
-         return {"int": 42, "str": "test"}
+     def test_values() -> {"width": int, "description": str}:
+         return {"width": 42, "description": "test"}
+
+* ``TypeForm``: this feature enables ``TypeForm``, as described in
+  `PEP 747 – Annotating Type Forms <https://peps.python.org/pep-0747/>_`.
 
 
 Miscellaneous

docs/source/duck_type_compatibility.rst

@@ -9,6 +9,8 @@ supported for a small set of built-in types:
 * ``int`` is duck type compatible with ``float`` and ``complex``.
 * ``float`` is duck type compatible with ``complex``.
 * ``bytearray`` and ``memoryview`` are duck type compatible with ``bytes``.
+  (this will be disabled by default in **mypy 2.0**, and currently can be
+  disabled with :option:`--strict-bytes <mypy --strict-bytes>`.)
 
 For example, mypy considers an ``int`` object to be valid whenever a
 ``float`` object is expected.  Thus code like this is nice and clean

docs/source/error_code_list.rst

@@ -1286,6 +1286,65 @@ type must be a subtype of the original type::
     def g(x: object) -> TypeIs[str]:  # OK
         ...
 
+.. _code-maybe-unrecognized-str-typeform:
+
+String appears in a context which expects a TypeForm [maybe-unrecognized-str-typeform]
+--------------------------------------------------------------------------------------
+
+TypeForm literals may contain string annotations:
+
+.. code-block:: python
+
+   typx1: TypeForm = str | None
+   typx2: TypeForm = 'str | None'  # OK
+   typx3: TypeForm = 'str' | None  # OK
+
+However TypeForm literals containing a string annotation can only be recognized
+by mypy in the following locations:
+
+.. code-block:: python
+
+   typx_var: TypeForm = 'str | None'  # assignment r-value
+
+   def func(typx_param: TypeForm) -> TypeForm:
+       return 'str | None'  # returned expression
+
+   func('str | None')  # callable's argument
+
+If you try to use a string annotation in some other location
+which expects a TypeForm, the string value will always be treated as a ``str``
+even if a ``TypeForm`` would be more appropriate and this error code
+will be generated:
+
+.. code-block:: python
+
+   # Error: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize.  [maybe-unrecognized-str-typeform]
+   # Error: List item 0 has incompatible type "str"; expected "TypeForm[Any]"  [list-item]
+   list_of_typx: list[TypeForm] = ['str | None', float]
+
+Fix the error by surrounding the entire type with ``TypeForm(...)``:
+
+.. code-block:: python
+
+   list_of_typx: list[TypeForm] = [TypeForm('str | None'), float]  # OK
+
+Similarly, if you try to use a string literal in a location which expects a
+TypeForm, this error code will be generated:
+
+.. code-block:: python
+
+   dict_of_typx = {'str_or_none': TypeForm(str | None)}
+   # Error: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize.  [maybe-unrecognized-str-typeform]
+   list_of_typx: list[TypeForm] = [dict_of_typx['str_or_none']]
+
+Fix the error by adding ``# type: ignore[maybe-unrecognized-str-typeform]``
+to the line with the string literal:
+
+.. code-block:: python
+
+   dict_of_typx = {'str_or_none': TypeForm(str | None)}
+   list_of_typx: list[TypeForm] = [dict_of_typx['str_or_none']]  # type: ignore[maybe-unrecognized-str-typeform]
+
 .. _code-misc:
 
 Miscellaneous checks [misc]

docs/source/kinds_of_types.rst

@@ -294,6 +294,7 @@ isn't supported by the runtime with some limitations, if you use
    def f(x: int | str) -> None:   # OK on Python 3.7 and later
        ...
 
+.. _no-strict-optional:
 .. _strict_optional:
 
 Optional types and the None type

docs/source/typed_dict.rst

@@ -303,8 +303,8 @@ to use inline TypedDict syntax. For example:
 
 .. code-block:: python
 
-    def test_values() -> {"int": int, "str": str}:
-        return {"int": 42, "str": "test"}
+    def test_values() -> {"width": int, "description": str}:
+        return {"width": 42, "description": "test"}
 
     class Response(TypedDict):
         status: int

misc/analyze_typeform_stats.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""
+Analyze TypeForm parsing efficiency from mypy build stats.
+
+Usage:
+    python3 analyze_typeform_stats.py '<mypy_output_with_stats>'
+    python3 -m mypy --dump-build-stats file.py 2>&1 | python3 analyze_typeform_stats.py
+
+Example output:
+    TypeForm Expression Parsing Statistics:
+    ==================================================
+    Total calls to SA.try_parse_as_type_expression: 14,555
+    Quick rejections (no full parse): 14,255
+    Full parses attempted: 300
+    - Successful: 248
+    - Failed: 52
+
+    Efficiency Metrics:
+    - Quick rejection rate: 97.9%
+    - Full parse rate: 2.1%
+    - Full parse success rate: 82.7%
+    - Overall success rate: 1.7%
+
+    Performance Implications:
+    - Expensive failed full parses: 52 (0.4% of all calls)
+
+See also:
+    - mypy/semanal.py: SemanticAnalyzer.try_parse_as_type_expression()
+    - mypy/semanal.py: DEBUG_TYPE_EXPRESSION_FULL_PARSE_FAILURES
+"""
+
+import re
+import sys
+
+
+def analyze_stats(output: str) -> None:
+    """Parse mypy stats output and calculate TypeForm parsing efficiency."""
+
+    # Extract the three counters
+    total_match = re.search(r"type_expression_parse_count:\s*(\d+)", output)
+    success_match = re.search(r"type_expression_full_parse_success_count:\s*(\d+)", output)
+    failure_match = re.search(r"type_expression_full_parse_failure_count:\s*(\d+)", output)
+
+    if not (total_match and success_match and failure_match):
+        print("Error: Could not find all required counters in output")
+        return
+
+    total = int(total_match.group(1))
+    successes = int(success_match.group(1))
+    failures = int(failure_match.group(1))
+
+    full_parses = successes + failures
+
+    print("TypeForm Expression Parsing Statistics:")
+    print("=" * 50)
+    print(f"Total calls to SA.try_parse_as_type_expression: {total:,}")
+    print(f"Quick rejections (no full parse): {total - full_parses:,}")
+    print(f"Full parses attempted: {full_parses:,}")
+    print(f"  - Successful: {successes:,}")
+    print(f"  - Failed: {failures:,}")
+    if total > 0:
+        print()
+        print("Efficiency Metrics:")
+        print(f"  - Quick rejection rate: {((total - full_parses) / total * 100):.1f}%")
+        print(f"  - Full parse rate: {(full_parses / total * 100):.1f}%")
+        print(f"  - Full parse success rate: {(successes / full_parses * 100):.1f}%")
+        print(f"  - Overall success rate: {(successes / total * 100):.1f}%")
+        print()
+        print("Performance Implications:")
+        print(
+            f"  - Expensive failed full parses: {failures:,} ({(failures / total * 100):.1f}% of all calls)"
+        )
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 1:
+        # Read from stdin
+        output = sys.stdin.read()
+    elif len(sys.argv) == 2:
+        # Read from command line argument
+        output = sys.argv[1]
+    else:
+        print("Usage: python3 analyze_typeform_stats.py [mypy_output_with_stats]")
+        print("Examples:")
+        print(
+            "  python3 -m mypy --dump-build-stats file.py 2>&1 | python3 analyze_typeform_stats.py"
+        )
+        print("  python3 analyze_typeform_stats.py 'output_string'")
+        sys.exit(1)
+
+    analyze_stats(output)

mypy-requirements.txt

@@ -4,4 +4,4 @@ typing_extensions>=4.6.0
 mypy_extensions>=1.0.0
 pathspec>=0.9.0
 tomli>=1.1.0; python_version<'3.11'
-librt>=0.3.0
+librt>=0.4.0

mypy/build.py

@@ -28,8 +28,10 @@
 from typing import TYPE_CHECKING, Any, Callable, ClassVar, Final, NoReturn, TextIO, TypedDict
 from typing_extensions import TypeAlias as _TypeAlias
 
+from librt.internal import cache_version
+
 import mypy.semanal_main
-from mypy.cache import Buffer, CacheMeta
+from mypy.cache import CACHE_VERSION, Buffer, CacheMeta
 from mypy.checker import TypeChecker
 from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter
 from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error
@@ -601,6 +603,7 @@ def __init__(
         self.options = options
         self.version_id = version_id
         self.modules: dict[str, MypyFile] = {}
+        self.import_map: dict[str, set[str]] = {}
         self.missing_modules: set[str] = set()
         self.fg_deps_meta: dict[str, FgDepMeta] = {}
         # fg_deps holds the dependencies of every module that has been
@@ -621,6 +624,7 @@ def __init__(
             self.incomplete_namespaces,
             self.errors,
             self.plugin,
+            self.import_map,
         )
         self.all_types: dict[Expression, Type] = {}  # Enabled by export_types
         self.indirection_detector = TypeIndirectionVisitor()
@@ -740,6 +744,26 @@ def getmtime(self, path: str) -> int:
         else:
             return int(self.metastore.getmtime(path))
 
+    def correct_rel_imp(self, file: MypyFile, imp: ImportFrom | ImportAll) -> str:
+        """Function to correct for relative imports."""
+        file_id = file.fullname
+        rel = imp.relative
+        if rel == 0:
+            return imp.id
+        if os.path.basename(file.path).startswith("__init__."):
+            rel -= 1
+        if rel != 0:
+            file_id = ".".join(file_id.split(".")[:-rel])
+        new_id = file_id + "." + imp.id if imp.id else file_id
+
+        if not new_id:
+            self.errors.set_file(file.path, file.name, self.options)
+            self.errors.report(
+                imp.line, 0, "No parent module -- cannot perform relative import", blocker=True
+            )
+
+        return new_id
+
     def all_imported_modules_in_file(self, file: MypyFile) -> list[tuple[int, str, int]]:
         """Find all reachable import statements in a file.
 
@@ -748,27 +772,6 @@ def all_imported_modules_in_file(self, file: MypyFile) -> list[tuple[int, str, i
 
         Can generate blocking errors on bogus relative imports.
         """
-
-        def correct_rel_imp(imp: ImportFrom | ImportAll) -> str:
-            """Function to correct for relative imports."""
-            file_id = file.fullname
-            rel = imp.relative
-            if rel == 0:
-                return imp.id
-            if os.path.basename(file.path).startswith("__init__."):
-                rel -= 1
-            if rel != 0:
-                file_id = ".".join(file_id.split(".")[:-rel])
-            new_id = file_id + "." + imp.id if imp.id else file_id
-
-            if not new_id:
-                self.errors.set_file(file.path, file.name, self.options)
-                self.errors.report(
-                    imp.line, 0, "No parent module -- cannot perform relative import", blocker=True
-                )
-
-            return new_id
-
         res: list[tuple[int, str, int]] = []
         for imp in file.imports:
             if not imp.is_unreachable:
@@ -783,7 +786,7 @@ def correct_rel_imp(imp: ImportFrom | ImportAll) -> str:
                             ancestors.append(part)
                             res.append((ancestor_pri, ".".join(ancestors), imp.line))
                 elif isinstance(imp, ImportFrom):
-                    cur_id = correct_rel_imp(imp)
+                    cur_id = self.correct_rel_imp(file, imp)
                     all_are_submodules = True
                     # Also add any imported names that are submodules.
                     pri = import_priority(imp, PRI_MED)
@@ -803,7 +806,7 @@ def correct_rel_imp(imp: ImportFrom | ImportAll) -> str:
                     res.append((pri, cur_id, imp.line))
                 elif isinstance(imp, ImportAll):
                     pri = import_priority(imp, PRI_HIGH)
-                    res.append((pri, correct_rel_imp(imp), imp.line))
+                    res.append((pri, self.correct_rel_imp(file, imp), imp.line))
 
         # Sort such that module (e.g. foo.bar.baz) comes before its ancestors (e.g. foo
         # and foo.bar) so that, if FindModuleCache finds the target module in a
@@ -1334,12 +1337,18 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No
             return None
     t1 = time.time()
     if isinstance(meta, bytes):
-        data_io = Buffer(meta)
+        # If either low-level buffer format or high-level cache layout changed, we
+        # cannot use the cache files, even with --skip-version-check.
+        # TODO: switch to something like librt.internal.read_byte() if this is slow.
+        if meta[0] != cache_version() or meta[1] != CACHE_VERSION:
+            manager.log(f"Metadata abandoned for {id}: incompatible cache format")
+            return None
+        data_io = Buffer(meta[2:])
         m = CacheMeta.read(data_io, data_file)
     else:
         m = CacheMeta.deserialize(meta, data_file)
     if m is None:
-        manager.log(f"Metadata abandoned for {id}: attributes are missing")
+        manager.log(f"Metadata abandoned for {id}: cannot deserialize data")
         return None
     t2 = time.time()
     manager.add_stats(
@@ -1671,7 +1680,9 @@ def write_cache_meta(meta: CacheMeta, manager: BuildManager, meta_file: str) ->
     if manager.options.fixed_format_cache:
         data_io = Buffer()
         meta.write(data_io)
-        meta_bytes = data_io.getvalue()
+        # Prefix with both low- and high-level cache format versions for future validation.
+        # TODO: switch to something like librt.internal.write_byte() if this is slow.
+        meta_bytes = bytes([cache_version(), CACHE_VERSION]) + data_io.getvalue()
     else:
         meta_dict = meta.serialize()
         meta_bytes = json_dumps(meta_dict, manager.options.debug_cache)
@@ -2888,6 +2899,9 @@ def dispatch(sources: list[BuildSource], manager: BuildManager, stdout: TextIO)
         manager.cache_enabled = False
         graph = load_graph(sources, manager)
 
+    for id in graph:
+        manager.import_map[id] = set(graph[id].dependencies + graph[id].suppressed)
+
     t1 = time.time()
     manager.add_stats(
         graph_size=len(graph),