Merge remote-tracking branch 'upstream/main' into typeform

Author: JelleZijlstra

.github/workflows/conformance.yml

@@ -32,6 +32,11 @@ jobs:
           uv sync --python 3.12 --frozen
           uv run --python 3.12 --frozen python src/main.py
 
+      - name: Validate conformance invariants
+        working-directory: conformance
+        run: |
+          uv run --python 3.12 --frozen python src/validate_results.py
+
       - name: Assert conformance results are up to date
         run: |
           if [ -n "$(git status --porcelain -- conformance/results)" ]; then

conformance/README.md

@@ -81,7 +81,7 @@ Note that some type checkers may not run on some platforms. If a type checker fa
 
 Different type checkers report errors in different ways (with different wording in error messages and different line numbers or character ranges for errors). This variation makes it difficult to fully automate test validation given that tests will want to check for both false positive and false negative type errors. Some level of manual inspection will therefore be needed to determine whether a type checker is fully conformant with all tests in any given test file. This "scoring" process is required only when the output of a test changes — e.g. when a new version of that type checker is released and the tests are rerun. We assume that the output of a type checker will be the same from one run to the next unless/until a new version is released that fixes or introduces a bug. In this case, the output will need to be manually inspected and the conformance results re-scored for those tests whose output has changed.
 
-Conformance results are reported and summarized for each supported type checker. Currently, results are reported for mypy, pyrefly, pyright, and zuban. It is the goal and desire to add additional type checkers over time.
+Conformance results are reported and summarized for each supported type checker. Currently, results are reported for mypy, pyrefly, pyright, zuban and ty. It is the goal and desire to add additional type checkers over time.
 
 ## Adding a New Test Case
 

conformance/pyproject.toml

@@ -9,6 +9,7 @@ dependencies = [
     "tomli",
     "tomlkit",
     "zuban",
+    "ty",
 ]
 
 [tool.uv]

conformance/results/mypy/constructors_call_metaclass.toml

@@ -1,4 +1,4 @@
-conformant = "Unupported"
+conformant = "Unsupported"
 notes = """
 Does not honor metaclass __call__ method when evaluating constructor call.
 Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

conformance/results/mypy/dataclasses_hash.toml

@@ -1,11 +1,19 @@
-conformant = "Partial"
+conformant = "Unsupported"
 notes = """
+Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.
+Does not report when an unhashable dataclass has `__hash__` called directly on an instance.
 Does not report when dataclass is not compatible with Hashable protocol.
 """
 output = """
+dataclasses_hash.py:14: error: Expression is of type "Callable[[object], int]", not "None"  [assert-type]
+dataclasses_hash.py:36: error: Expression is of type "Callable[[object], int]", not "None"  [assert-type]
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 15: Expected 1 errors
-Line 32: Expected 1 errors
+Line 17: Expected 1 errors
+Line 18: Expected 1 errors
+Line 39: Expected 1 errors
+Line 40: Expected 1 errors
+Line 14: Unexpected errors ['dataclasses_hash.py:14: error: Expression is of type "Callable[[object], int]", not "None"  [assert-type]']
+Line 36: Unexpected errors ['dataclasses_hash.py:36: error: Expression is of type "Callable[[object], int]", not "None"  [assert-type]']
 """

conformance/results/mypy/directives_version_platform.toml

@@ -4,10 +4,11 @@ Does not understand three-element form of sys.version checks.
 Does not understand os.name checks.
 """
 output = """
-directives_version_platform.py:19: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
-directives_version_platform.py:27: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
-directives_version_platform.py:40: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
-directives_version_platform.py:45: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
+directives_version_platform.py:26: error: Expression is of type "int | str", not "int"  [assert-type]
+directives_version_platform.py:33: error: Name "val3" is not defined  [name-defined]
+directives_version_platform.py:42: error: Expression is of type "int | str", not "int"  [assert-type]
+directives_version_platform.py:50: error: Name "val6" is not defined  [name-defined]
+directives_version_platform.py:59: error: Name "val9" is not defined  [name-defined]
 """
 conformance_automated = "Pass"
 errors_diff = """

conformance/results/mypy/generics_defaults.toml

@@ -1,38 +1,24 @@
 conformant = "Partial"
+notes = """
+Does not detect a TypeVar with a default used after a TypeVarTuple.
+Does not fully support defaults on TypeVarTuple and ParamSpec.
+"""
 output = """
 generics_defaults.py:24: error: "T" cannot appear after "DefaultStrT" in type parameter list because it has no default type  [misc]
-generics_defaults.py:30: error: Expression is of type "type[NoNonDefaults[DefaultStrT, DefaultIntT]]", not "type[NoNonDefaults[str, int]]"  [assert-type]
-generics_defaults.py:31: error: Expression is of type "type[NoNonDefaults[str, DefaultIntT]]", not "type[NoNonDefaults[str, int]]"  [assert-type]
-generics_defaults.py:38: error: Expression is of type "type[OneDefault[float, DefaultBoolT]]", not "type[OneDefault[float, bool]]"  [assert-type]
-generics_defaults.py:45: error: Expression is of type "type[AllTheDefaults[T1, T2, DefaultStrT, DefaultIntT, DefaultBoolT]]", not "type[AllTheDefaults[Any, Any, str, int, bool]]"  [assert-type]
-generics_defaults.py:46: error: Expression is of type "type[AllTheDefaults[int, complex, DefaultStrT, DefaultIntT, DefaultBoolT]]", not "type[AllTheDefaults[int, complex, str, int, bool]]"  [assert-type]
-generics_defaults.py:50: error: Type application has too few types (expected between 2 and 5)  [misc]
-generics_defaults.py:52: error: Expression is of type "type[AllTheDefaults[int, complex, DefaultStrT, DefaultIntT, DefaultBoolT]]", not "type[AllTheDefaults[int, complex, str, int, bool]]"  [assert-type]
-generics_defaults.py:55: error: Expression is of type "type[AllTheDefaults[int, complex, str, DefaultIntT, DefaultBoolT]]", not "type[AllTheDefaults[int, complex, str, int, bool]]"  [assert-type]
-generics_defaults.py:59: error: Expression is of type "type[AllTheDefaults[int, complex, str, int, DefaultBoolT]]", not "type[AllTheDefaults[int, complex, str, int, bool]]"  [assert-type]
-generics_defaults.py:79: error: Expression is of type "type[Class_ParamSpec[DefaultP]]", not "type[Class_ParamSpec[[str, int]]]"  [assert-type]
-generics_defaults.py:94: error: Expression is of type "type[Class_TypeVarTuple[*DefaultTs]]", not "type[Class_TypeVarTuple[str, int]]"  [assert-type]
-generics_defaults.py:107: error: TypeVar default must be a subtype of the bound type  [misc]
-generics_defaults.py:114: error: TypeVar default must be one of the constraint types  [misc]
-generics_defaults.py:132: error: Expression is of type "int", not "Any"  [assert-type]
-generics_defaults.py:156: error: Expression is of type "type[Foo6[*tuple[Any, ...], Any]]", not "type[Foo6[int, str, [float, bool]]]"  [assert-type]
-generics_defaults.py:156: error: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "str"  [misc]
-generics_defaults.py:157: error: Expression is of type "type[Foo6[*tuple[Any, ...], Any]]", not "type[Foo6[int, str, [bytes]]]"  [assert-type]
-generics_defaults.py:157: error: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "str"  [misc]
+generics_defaults.py:66: error: "AllTheDefaults" expects between 2 and 5 type arguments, but 1 given  [type-arg]
+generics_defaults.py:139: error: Expression is of type "tuple[*tuple[str, int], ...]", not "tuple[str, int]"  [assert-type]
+generics_defaults.py:152: error: TypeVar default must be a subtype of the bound type  [misc]
+generics_defaults.py:159: error: TypeVar default must be one of the constraint types  [misc]
+generics_defaults.py:177: error: Expression is of type "int", not "Any"  [assert-type]
+generics_defaults.py:203: error: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "str"  [valid-type]
+generics_defaults.py:204: error: Expression is of type "tuple[Any, ...]", not "tuple[int, str]"  [assert-type]
+generics_defaults.py:205: error: Expression is of type "def (*Any, **Any) -> None", not "Callable[[float, bool], None]"  [assert-type]
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 143: Expected 1 errors
-Line 30: Unexpected errors ['generics_defaults.py:30: error: Expression is of type "type[NoNonDefaults[DefaultStrT, DefaultIntT]]", not "type[NoNonDefaults[str, int]]"  [assert-type]']
-Line 31: Unexpected errors ['generics_defaults.py:31: error: Expression is of type "type[NoNonDefaults[str, DefaultIntT]]", not "type[NoNonDefaults[str, int]]"  [assert-type]']
-Line 38: Unexpected errors ['generics_defaults.py:38: error: Expression is of type "type[OneDefault[float, DefaultBoolT]]", not "type[OneDefault[float, bool]]"  [assert-type]']
-Line 45: Unexpected errors ['generics_defaults.py:45: error: Expression is of type "type[AllTheDefaults[T1, T2, DefaultStrT, DefaultIntT, DefaultBoolT]]", not "type[AllTheDefaults[Any, Any, str, int, bool]]"  [assert-type]']
-Line 46: Unexpected errors ['generics_defaults.py:46: error: Expression is of type "type[AllTheDefaults[int, complex, DefaultStrT, DefaultIntT, DefaultBoolT]]", not "type[AllTheDefaults[int, complex, str, int, bool]]"  [assert-type]']
-Line 52: Unexpected errors ['generics_defaults.py:52: error: Expression is of type "type[AllTheDefaults[int, complex, DefaultStrT, DefaultIntT, DefaultBoolT]]", not "type[AllTheDefaults[int, complex, str, int, bool]]"  [assert-type]']
-Line 55: Unexpected errors ['generics_defaults.py:55: error: Expression is of type "type[AllTheDefaults[int, complex, str, DefaultIntT, DefaultBoolT]]", not "type[AllTheDefaults[int, complex, str, int, bool]]"  [assert-type]']
-Line 59: Unexpected errors ['generics_defaults.py:59: error: Expression is of type "type[AllTheDefaults[int, complex, str, int, DefaultBoolT]]", not "type[AllTheDefaults[int, complex, str, int, bool]]"  [assert-type]']
-Line 79: Unexpected errors ['generics_defaults.py:79: error: Expression is of type "type[Class_ParamSpec[DefaultP]]", not "type[Class_ParamSpec[[str, int]]]"  [assert-type]']
-Line 94: Unexpected errors ['generics_defaults.py:94: error: Expression is of type "type[Class_TypeVarTuple[*DefaultTs]]", not "type[Class_TypeVarTuple[str, int]]"  [assert-type]']
-Line 156: Unexpected errors ['generics_defaults.py:156: error: Expression is of type "type[Foo6[*tuple[Any, ...], Any]]", not "type[Foo6[int, str, [float, bool]]]"  [assert-type]', 'generics_defaults.py:156: error: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "str"  [misc]']
-Line 157: Unexpected errors ['generics_defaults.py:157: error: Expression is of type "type[Foo6[*tuple[Any, ...], Any]]", not "type[Foo6[int, str, [bytes]]]"  [assert-type]', 'generics_defaults.py:157: error: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "str"  [misc]']
+Line 188: Expected 1 errors
+Line 139: Unexpected errors ['generics_defaults.py:139: error: Expression is of type "tuple[*tuple[str, int], ...]", not "tuple[str, int]"  [assert-type]']
+Line 203: Unexpected errors ['generics_defaults.py:203: error: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "str"  [valid-type]']
+Line 204: Unexpected errors ['generics_defaults.py:204: error: Expression is of type "tuple[Any, ...]", not "tuple[int, str]"  [assert-type]']
+Line 205: Unexpected errors ['generics_defaults.py:205: error: Expression is of type "def (*Any, **Any) -> None", not "Callable[[float, bool], None]"  [assert-type]']
 """

conformance/results/mypy/generics_defaults_referential.toml

@@ -1,23 +1,22 @@
 conformant = "Partial"
+notes = """
+Does not correctly handle defaults referencing other TypeVars.
+"""
 output = """
 generics_defaults_referential.py:23: error: Expression is of type "type[slice[StartT, StopT, StepT]]", not "type[slice[int, int, int | None]]"  [assert-type]
-generics_defaults_referential.py:37: error: Argument 1 to "Foo" has incompatible type "str"; expected "int"  [arg-type]
-generics_defaults_referential.py:53: error: Type parameter "Start2T" has a default type that refers to one or more type variables that are out of scope  [misc]
-generics_defaults_referential.py:74: error: TypeVar default must be one of the constraint types  [misc]
-generics_defaults_referential.py:77: error: TypeVar default must be one of the constraint types  [misc]
+generics_defaults_referential.py:38: error: Argument 1 to "Foo" has incompatible type "str"; expected "int"  [arg-type]
+generics_defaults_referential.py:54: error: Type parameter "Start2T" has a default type that refers to one or more type variables that are out of scope  [misc]
+generics_defaults_referential.py:75: error: TypeVar default must be one of the constraint types  [misc]
 generics_defaults_referential.py:78: error: TypeVar default must be one of the constraint types  [misc]
-generics_defaults_referential.py:94: error: Expression is of type "type[Bar[Z1, ListDefaultT]]", not "type[Bar[Any, list[Any]]]"  [assert-type]
-generics_defaults_referential.py:95: error: Expression is of type "type[Bar[int, ListDefaultT]]", not "type[Bar[int, list[int]]]"  [assert-type]
-generics_defaults_referential.py:96: error: Expression is of type "Bar[int, list[Never]]", not "Bar[int, list[int]]"  [assert-type]
+generics_defaults_referential.py:79: error: TypeVar default must be one of the constraint types  [misc]
+generics_defaults_referential.py:104: error: Expression is of type "Bar[int, list[Never]]", not "Bar[int, list[int]]"  [assert-type]
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 36: Expected 1 errors
-Line 60: Expected 1 errors
-Line 68: Expected 1 errors
+Line 37: Expected 1 errors
+Line 61: Expected 1 errors
+Line 69: Expected 1 errors
 Line 23: Unexpected errors ['generics_defaults_referential.py:23: error: Expression is of type "type[slice[StartT, StopT, StepT]]", not "type[slice[int, int, int | None]]"  [assert-type]']
-Line 77: Unexpected errors ['generics_defaults_referential.py:77: error: TypeVar default must be one of the constraint types  [misc]']
-Line 94: Unexpected errors ['generics_defaults_referential.py:94: error: Expression is of type "type[Bar[Z1, ListDefaultT]]", not "type[Bar[Any, list[Any]]]"  [assert-type]']
-Line 95: Unexpected errors ['generics_defaults_referential.py:95: error: Expression is of type "type[Bar[int, ListDefaultT]]", not "type[Bar[int, list[int]]]"  [assert-type]']
-Line 96: Unexpected errors ['generics_defaults_referential.py:96: error: Expression is of type "Bar[int, list[Never]]", not "Bar[int, list[int]]"  [assert-type]']
+Line 78: Unexpected errors ['generics_defaults_referential.py:78: error: TypeVar default must be one of the constraint types  [misc]']
+Line 104: Unexpected errors ['generics_defaults_referential.py:104: error: Expression is of type "Bar[int, list[Never]]", not "Bar[int, list[int]]"  [assert-type]']
 """

conformance/results/mypy/generics_defaults_specialization.toml

@@ -1,4 +1,7 @@
 conformant = "Partial"
+notes = """
+Does not correctly resolve defaults when classes are used directly.
+"""
 output = """
 generics_defaults_specialization.py:30: error: Bad number of arguments for type alias, expected between 0 and 1, given 2  [type-arg]
 generics_defaults_specialization.py:45: error: Expression is of type "type[Bar[DefaultStrT]]", not "type[Bar[str]]"  [assert-type]

conformance/results/pyrefly/dataclasses_hash.toml

@@ -3,6 +3,8 @@ conformance_automated = "Pass"
 errors_diff = """
 """
 output = """
-ERROR dataclasses_hash.py:15:16-22: `DC1` is not assignable to `Hashable` [bad-assignment]
-ERROR dataclasses_hash.py:32:16-22: `DC3` is not assignable to `Hashable` [bad-assignment]
+ERROR dataclasses_hash.py:17:1-16: Expected a callable, got `None` [not-callable]
+ERROR dataclasses_hash.py:18:16-22: `DC1` is not assignable to `Hashable` [bad-assignment]
+ERROR dataclasses_hash.py:39:1-16: Expected a callable, got `None` [not-callable]
+ERROR dataclasses_hash.py:40:16-22: `DC3` is not assignable to `Hashable` [bad-assignment]
 """