Merge remote-tracking branch 'upstream/main' into tvt-match

Author: JelleZijlstra

.github/workflows/conformance.yml

@@ -32,6 +32,11 @@ jobs:
           uv sync --python 3.12 --frozen
           uv run --python 3.12 --frozen python src/main.py
 
+      - name: Validate conformance invariants
+        working-directory: conformance
+        run: |
+          uv run --python 3.12 --frozen python src/validate_results.py
+
       - name: Assert conformance results are up to date
         run: |
           if [ -n "$(git status --porcelain -- conformance/results)" ]; then

conformance/README.md

@@ -12,6 +12,7 @@ This project contains test cases for behaviors defined in the Python typing spec
 
 * [concepts](https://typing.python.org/en/latest/spec/concepts.html)
 * [annotations](https://typing.python.org/en/latest/spec/annotations.html)
+* [typeforms](https://typing.python.org/en/latest/spec/type-forms.html)
 * [specialtypes](https://typing.python.org/en/latest/spec/special-types.html)
 * [generics](https://typing.python.org/en/latest/spec/generics.html)
 * [qualifiers](https://typing.python.org/en/latest/spec/qualifiers.html)
@@ -80,7 +81,7 @@ Note that some type checkers may not run on some platforms. If a type checker fa
 
 Different type checkers report errors in different ways (with different wording in error messages and different line numbers or character ranges for errors). This variation makes it difficult to fully automate test validation given that tests will want to check for both false positive and false negative type errors. Some level of manual inspection will therefore be needed to determine whether a type checker is fully conformant with all tests in any given test file. This "scoring" process is required only when the output of a test changes — e.g. when a new version of that type checker is released and the tests are rerun. We assume that the output of a type checker will be the same from one run to the next unless/until a new version is released that fixes or introduces a bug. In this case, the output will need to be manually inspected and the conformance results re-scored for those tests whose output has changed.
 
-Conformance results are reported and summarized for each supported type checker. Currently, results are reported for mypy, pyrefly, pyright, and zuban. It is the goal and desire to add additional type checkers over time.
+Conformance results are reported and summarized for each supported type checker. Currently, results are reported for mypy, pyrefly, pyright, zuban and ty. It is the goal and desire to add additional type checkers over time.
 
 ## Adding a New Test Case
 

conformance/pyproject.toml

@@ -9,6 +9,7 @@ dependencies = [
     "tomli",
     "tomlkit",
     "zuban",
+    "ty",
 ]
 
 [tool.uv]

conformance/results/mypy/aliases_recursive.toml

@@ -3,7 +3,7 @@ output = """
 aliases_recursive.py:19: error: Dict entry 1 has incompatible type "str": "complex"; expected "str": "int | str | float | list[Json] | dict[str, Json] | None"  [dict-item]
 aliases_recursive.py:20: error: List item 1 has incompatible type "complex"; expected "int | str | float | list[Json] | dict[str, Json] | None"  [list-item]
 aliases_recursive.py:38: error: Incompatible types in assignment (expression has type "tuple[int, tuple[str, int], tuple[int, tuple[int, list[int]]]]", variable has type "RecursiveTuple")  [assignment]
-aliases_recursive.py:39: error: Name "t6" already defined on line 38  [no-redef]
+aliases_recursive.py:39: error: Incompatible types in assignment (expression has type "tuple[int, list[int]]", variable has type "RecursiveTuple")  [assignment]
 aliases_recursive.py:50: error: Dict entry 0 has incompatible type "str": "list[int]"; expected "str": "str | int | Mapping[str, RecursiveMapping]"  [dict-item]
 aliases_recursive.py:51: error: Dict entry 2 has incompatible type "str": "list[int]"; expected "str": "str | int | Mapping[str, RecursiveMapping]"  [dict-item]
 aliases_recursive.py:52: error: Dict entry 2 has incompatible type "str": "list[int]"; expected "str": "str | int | Mapping[str, RecursiveMapping]"  [dict-item]

conformance/results/mypy/constructors_call_metaclass.toml

@@ -1,4 +1,4 @@
-conformant = "Unupported"
+conformant = "Unsupported"
 notes = """
 Does not honor metaclass __call__ method when evaluating constructor call.
 Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

conformance/results/mypy/directives_version_platform.toml

@@ -4,10 +4,11 @@ Does not understand three-element form of sys.version checks.
 Does not understand os.name checks.
 """
 output = """
-directives_version_platform.py:19: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
-directives_version_platform.py:27: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
-directives_version_platform.py:40: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
-directives_version_platform.py:45: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
+directives_version_platform.py:26: error: Expression is of type "int | str", not "int"  [assert-type]
+directives_version_platform.py:33: error: Name "val3" is not defined  [name-defined]
+directives_version_platform.py:42: error: Expression is of type "int | str", not "int"  [assert-type]
+directives_version_platform.py:50: error: Name "val6" is not defined  [name-defined]
+directives_version_platform.py:59: error: Name "val9" is not defined  [name-defined]
 """
 conformance_automated = "Pass"
 errors_diff = """

conformance/results/mypy/enums_members.toml

@@ -9,31 +9,23 @@ Does not treat attributes with private names as non-members.
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 82: Expected 1 errors
-Line 83: Expected 1 errors
-Line 129: Expected 1 errors
-Line 27: Unexpected errors ['enums_members.py:27: error: Expression is of type "Literal[Pet.genus]", not "str"  [assert-type]']
-Line 28: Unexpected errors ['enums_members.py:28: error: Expression is of type "Literal[Pet.species]", not "str"  [assert-type]']
-Line 35: Unexpected errors ['enums_members.py:35: error: Expression is of type "Literal[Pet2.genus]", not "str"  [assert-type]']
-Line 36: Unexpected errors ['enums_members.py:36: error: Expression is of type "Literal[Pet2.species]", not "str"  [assert-type]']
-Line 100: Unexpected errors ['enums_members.py:100: error: Expression is of type "Literal[TrafficLight.AMBER]", not "Literal[TrafficLight.YELLOW]"  [assert-type]']
-Line 117: Unexpected errors ['enums_members.py:117: error: Expression is of type "member[Callable[[Example], None]]", not "Any"  [assert-type]', 'enums_members.py:117: error: Parameter 1 of Literal[...] is invalid  [valid-type]']
+Line 86: Expected 1 errors
+Line 87: Expected 1 errors
+Line 133: Expected 1 errors
+Line 104: Unexpected errors ['enums_members.py:104: error: Expression is of type "Literal[TrafficLight.AMBER]", not "Literal[TrafficLight.YELLOW]"  [assert-type]']
+Line 121: Unexpected errors ['enums_members.py:121: error: Expression is of type "member[Callable[[Example], None]]", not "Any"  [assert-type]', 'enums_members.py:121: error: Parameter 1 of Literal[...] is invalid  [valid-type]']
 """
 output = """
-enums_members.py:27: error: Expression is of type "Literal[Pet.genus]", not "str"  [assert-type]
-enums_members.py:28: error: Expression is of type "Literal[Pet.species]", not "str"  [assert-type]
-enums_members.py:35: error: Expression is of type "Literal[Pet2.genus]", not "str"  [assert-type]
-enums_members.py:36: error: Expression is of type "Literal[Pet2.species]", not "str"  [assert-type]
-enums_members.py:50: error: Enum members must be left unannotated  [misc]
-enums_members.py:50: note: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members
-enums_members.py:84: error: Parameter 1 of Literal[...] is invalid  [valid-type]
-enums_members.py:85: error: Parameter 1 of Literal[...] is invalid  [valid-type]
-enums_members.py:100: error: Expression is of type "Literal[TrafficLight.AMBER]", not "Literal[TrafficLight.YELLOW]"  [assert-type]
-enums_members.py:116: error: Expression is of type "int", not "Literal[Example.b]"  [assert-type]
-enums_members.py:117: error: Expression is of type "member[Callable[[Example], None]]", not "Any"  [assert-type]
-enums_members.py:117: error: Parameter 1 of Literal[...] is invalid  [valid-type]
-enums_members.py:128: note: Revealed type is "Any"
-enums_members.py:128: note: 'reveal_type' always outputs 'Any' in unchecked functions
-enums_members.py:146: error: Expression is of type "Literal[Pet5.DOG]", not "int"  [assert-type]
-enums_members.py:147: error: Expression is of type "Literal[Pet5.FISH]", not "int"  [assert-type]
+enums_members.py:54: error: Enum members must be left unannotated  [misc]
+enums_members.py:54: note: See https://typing.readthedocs.io/en/latest/spec/enums.html#defining-members
+enums_members.py:88: error: Parameter 1 of Literal[...] is invalid  [valid-type]
+enums_members.py:89: error: Parameter 1 of Literal[...] is invalid  [valid-type]
+enums_members.py:104: error: Expression is of type "Literal[TrafficLight.AMBER]", not "Literal[TrafficLight.YELLOW]"  [assert-type]
+enums_members.py:120: error: Expression is of type "int", not "Literal[Example.b]"  [assert-type]
+enums_members.py:121: error: Expression is of type "member[Callable[[Example], None]]", not "Any"  [assert-type]
+enums_members.py:121: error: Parameter 1 of Literal[...] is invalid  [valid-type]
+enums_members.py:132: note: Revealed type is "Any"
+enums_members.py:132: note: 'reveal_type' always outputs 'Any' in unchecked functions
+enums_members.py:150: error: Expression is of type "Literal[Pet5.DOG]", not "int"  [assert-type]
+enums_members.py:151: error: Expression is of type "Literal[Pet5.FISH]", not "int"  [assert-type]
 """

conformance/results/mypy/generics_defaults.toml

@@ -1,4 +1,8 @@
 conformant = "Partial"
+notes = """
+Does not detect a TypeVar with a default used after a TypeVarTuple.
+Does not fully support defaults on TypeVarTuple and ParamSpec.
+"""
 output = """
 generics_defaults.py:24: error: "T" cannot appear after "DefaultStrT" in type parameter list because it has no default type  [misc]
 generics_defaults.py:66: error: "AllTheDefaults" expects between 2 and 5 type arguments, but 1 given  [type-arg]

conformance/results/mypy/generics_defaults_referential.toml

@@ -1,19 +1,22 @@
 conformant = "Partial"
+notes = """
+Does not correctly handle defaults referencing other TypeVars.
+"""
 output = """
 generics_defaults_referential.py:23: error: Expression is of type "type[slice[StartT, StopT, StepT]]", not "type[slice[int, int, int | None]]"  [assert-type]
-generics_defaults_referential.py:37: error: Argument 1 to "Foo" has incompatible type "str"; expected "int"  [arg-type]
-generics_defaults_referential.py:53: error: Type parameter "Start2T" has a default type that refers to one or more type variables that are out of scope  [misc]
-generics_defaults_referential.py:74: error: TypeVar default must be one of the constraint types  [misc]
-generics_defaults_referential.py:77: error: TypeVar default must be one of the constraint types  [misc]
+generics_defaults_referential.py:38: error: Argument 1 to "Foo" has incompatible type "str"; expected "int"  [arg-type]
+generics_defaults_referential.py:54: error: Type parameter "Start2T" has a default type that refers to one or more type variables that are out of scope  [misc]
+generics_defaults_referential.py:75: error: TypeVar default must be one of the constraint types  [misc]
 generics_defaults_referential.py:78: error: TypeVar default must be one of the constraint types  [misc]
-generics_defaults_referential.py:103: error: Expression is of type "Bar[int, list[Never]]", not "Bar[int, list[int]]"  [assert-type]
+generics_defaults_referential.py:79: error: TypeVar default must be one of the constraint types  [misc]
+generics_defaults_referential.py:104: error: Expression is of type "Bar[int, list[Never]]", not "Bar[int, list[int]]"  [assert-type]
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 36: Expected 1 errors
-Line 60: Expected 1 errors
-Line 68: Expected 1 errors
+Line 37: Expected 1 errors
+Line 61: Expected 1 errors
+Line 69: Expected 1 errors
 Line 23: Unexpected errors ['generics_defaults_referential.py:23: error: Expression is of type "type[slice[StartT, StopT, StepT]]", not "type[slice[int, int, int | None]]"  [assert-type]']
-Line 77: Unexpected errors ['generics_defaults_referential.py:77: error: TypeVar default must be one of the constraint types  [misc]']
-Line 103: Unexpected errors ['generics_defaults_referential.py:103: error: Expression is of type "Bar[int, list[Never]]", not "Bar[int, list[int]]"  [assert-type]']
+Line 78: Unexpected errors ['generics_defaults_referential.py:78: error: TypeVar default must be one of the constraint types  [misc]']
+Line 104: Unexpected errors ['generics_defaults_referential.py:104: error: Expression is of type "Bar[int, list[Never]]", not "Bar[int, list[int]]"  [assert-type]']
 """

conformance/results/mypy/generics_defaults_specialization.toml

@@ -1,10 +1,12 @@
 conformant = "Partial"
+notes = """
+Does not correctly resolve defaults when classes are used directly.
+"""
 output = """
 generics_defaults_specialization.py:30: error: Bad number of arguments for type alias, expected between 0 and 1, given 2  [type-arg]
-generics_defaults_specialization.py:45: error: Expression is of type "type[Bar[DefaultStrT]]", not "type[Bar[str]]"  [assert-type]
-generics_defaults_specialization.py:55: error: The type "type[Foo]" is not generic and not indexable  [misc]
+generics_defaults_specialization.py:56: error: The type "type[Foo]" is not generic and not indexable  [misc]
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 45: Unexpected errors ['generics_defaults_specialization.py:45: error: Expression is of type "type[Bar[DefaultStrT]]", not "type[Bar[str]]"  [assert-type]']
+Line 46: Expected 1 errors
 """