Add conformance results for ty (#2208)

Co-authored-by: Carl Meyer <carl@oddbird.net>

Author: AlexWaygood

conformance/README.md

@@ -80,7 +80,7 @@ Note that some type checkers may not run on some platforms. If a type checker fa
 
 Different type checkers report errors in different ways (with different wording in error messages and different line numbers or character ranges for errors). This variation makes it difficult to fully automate test validation given that tests will want to check for both false positive and false negative type errors. Some level of manual inspection will therefore be needed to determine whether a type checker is fully conformant with all tests in any given test file. This "scoring" process is required only when the output of a test changes — e.g. when a new version of that type checker is released and the tests are rerun. We assume that the output of a type checker will be the same from one run to the next unless/until a new version is released that fixes or introduces a bug. In this case, the output will need to be manually inspected and the conformance results re-scored for those tests whose output has changed.
 
-Conformance results are reported and summarized for each supported type checker. Currently, results are reported for mypy, pyrefly, pyright, and zuban. It is the goal and desire to add additional type checkers over time.
+Conformance results are reported and summarized for each supported type checker. Currently, results are reported for mypy, pyrefly, pyright, zuban and ty. It is the goal and desire to add additional type checkers over time.
 
 ## Adding a New Test Case
 

conformance/pyproject.toml

@@ -9,6 +9,7 @@ dependencies = [
     "tomli",
     "tomlkit",
     "zuban",
+    "ty",
 ]
 
 [tool.uv]

conformance/results/results.html

@@ -0,0 +1,30 @@
+conformance_automated = "Fail"
+conformant = "Partial"
+notes = "`Concatenate` in type aliases is currently unsupported"
+errors_diff = """
+Line 87: Expected 1 errors
+Line 100: Expected 1 errors
+Line 57: Unexpected errors ['aliases_explicit.py:57:5: error[type-assertion-failure] Type `Unknown` does not match asserted type `(int, str, str, /) -> None`']
+"""
+output = """
+aliases_explicit.py:57:5: error[type-assertion-failure] Type `Unknown` does not match asserted type `(int, str, str, /) -> None`
+aliases_explicit.py:67:9: error[not-subscriptable] Cannot subscript non-generic type `<types.UnionType special-form 'int | None'>`
+aliases_explicit.py:68:9: error[not-subscriptable] Cannot subscript non-generic type `<class 'list[int | None]'>`
+aliases_explicit.py:69:29: error[invalid-type-arguments] Too many type arguments: expected 1, got 2
+aliases_explicit.py:70:29: error[invalid-type-arguments] Too many type arguments: expected 1, got 2
+aliases_explicit.py:71:24: error[invalid-type-arguments] Type argument for `ParamSpec` must be either a list of types, `ParamSpec`, `Concatenate`, or `...`
+aliases_explicit.py:79:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:80:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:81:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:82:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:83:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:84:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:85:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:86:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:88:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:89:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:90:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:91:1: error[invalid-type-form] Invalid right-hand side for `typing.TypeAlias` assignment
+aliases_explicit.py:101:6: error[call-non-callable] Object of type `UnionType` is not callable
+aliases_explicit.py:102:5: error[not-subscriptable] Cannot subscript non-generic type `<types.UnionType special-form 'list[Unknown] | set[Unknown]'>`
+"""

conformance/results/ty/aliases_explicit.toml

@@ -0,0 +1,34 @@
+conformance_automated = "Fail"
+conformant = "Partial"
+notes = """
+Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.
+Does not support `Concatenate` in type aliases.
+"""
+errors_diff = """
+Line 106: Expected 1 errors
+Line 111: Expected 1 errors
+Line 112: Expected 1 errors
+Line 113: Expected 1 errors
+Line 117: Expected 1 errors
+Line 68: Unexpected errors ['aliases_implicit.py:68:5: error[type-assertion-failure] Type `Unknown` does not match asserted type `(int, str, str, /) -> None`']
+"""
+output = """
+aliases_implicit.py:68:5: error[type-assertion-failure] Type `Unknown` does not match asserted type `(int, str, str, /) -> None`
+aliases_implicit.py:76:9: error[not-subscriptable] Cannot subscript non-generic type `<types.UnionType special-form 'int | None'>`
+aliases_implicit.py:77:9: error[not-subscriptable] Cannot subscript non-generic type `<class 'list[int | None]'>`
+aliases_implicit.py:78:29: error[invalid-type-arguments] Too many type arguments: expected 1, got 2
+aliases_implicit.py:79:29: error[invalid-type-arguments] Too many type arguments: expected 1, got 2
+aliases_implicit.py:80:24: error[invalid-type-arguments] Type argument for `ParamSpec` must be either a list of types, `ParamSpec`, `Concatenate`, or `...`
+aliases_implicit.py:81:25: error[invalid-type-arguments] Type `str` is not assignable to upper bound `int | float` of type variable `TFloat@GoodTypeAlias12`
+aliases_implicit.py:107:9: error[invalid-type-form] Variable of type `list[<class 'int'> | <class 'str'>]` is not allowed in a type expression
+aliases_implicit.py:108:9: error[invalid-type-form] Variable of type `tuple[tuple[<class 'int'>, <class 'str'>]]` is not allowed in a type expression
+aliases_implicit.py:109:9: error[invalid-type-form] Variable of type `list[<class 'int'>]` is not allowed in a type expression
+aliases_implicit.py:110:9: error[invalid-type-form] Variable of type `dict[str, str]` is not allowed in a type expression
+aliases_implicit.py:114:9: error[invalid-type-form] Variable of type `Literal[3]` is not allowed in a type expression
+aliases_implicit.py:115:10: error[invalid-type-form] Variable of type `Literal[True]` is not allowed in a type expression
+aliases_implicit.py:116:10: error[invalid-type-form] Variable of type `Literal[1]` is not allowed in a type expression
+aliases_implicit.py:118:10: error[invalid-type-form] Variable of type `Literal["int"]` is not allowed in a type expression
+aliases_implicit.py:119:10: error[invalid-type-form] Variable of type `Literal["int | str"]` is not allowed in a type expression
+aliases_implicit.py:133:6: error[call-non-callable] Object of type `UnionType` is not callable
+aliases_implicit.py:135:5: error[not-subscriptable] Cannot subscript non-generic type `<types.UnionType special-form 'list[Unknown] | set[Unknown]'>`
+"""

conformance/results/ty/aliases_implicit.toml

@@ -0,0 +1,19 @@
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+aliases_newtype.py:11:8: error[invalid-argument-type] Argument is incorrect: Expected `int`, found `Literal["user"]`
+aliases_newtype.py:12:14: error[invalid-assignment] Object of type `Literal[42]` is not assignable to `UserId`
+aliases_newtype.py:18:11: error[invalid-assignment] Object of type `<NewType pseudo-class 'UserId'>` is not assignable to `type`
+aliases_newtype.py:23:16: error[invalid-argument-type] Argument to function `isinstance` is incorrect: Expected `type | UnionType | tuple[Divergent, ...]`, found `<NewType pseudo-class 'UserId'>`
+aliases_newtype.py:26:21: error[invalid-base] Cannot subclass an instance of NewType
+aliases_newtype.py:35:1: error[invalid-newtype] The name of a `NewType` (`BadName`) must match the name of the variable it is assigned to (`GoodName`)
+aliases_newtype.py:41:6: error[invalid-type-form] `GoodNewType1` is a `NewType` and cannot be specialized
+aliases_newtype.py:47:38: error[invalid-newtype] invalid base for `typing.NewType`: type `int | str`
+aliases_newtype.py:50:38: error[invalid-newtype] invalid base for `typing.NewType`: A `NewType` base cannot be generic
+aliases_newtype.py:52:38: error[invalid-newtype] invalid base for `typing.NewType`: type `Hashable`
+aliases_newtype.py:54:38: error[invalid-newtype] invalid base for `typing.NewType`: type `Literal[7]`
+aliases_newtype.py:61:38: error[invalid-newtype] invalid base for `typing.NewType`: type `TD1`
+aliases_newtype.py:63:15: error[invalid-newtype] Wrong number of arguments in `NewType` creation: expected 2, found 3
+aliases_newtype.py:65:38: error[invalid-newtype] invalid base for `typing.NewType`: type `Any`
+"""

conformance/results/ty/aliases_newtype.toml

@@ -0,0 +1,17 @@
+conformance_automated = "Fail"
+conformant = "Unsupported"
+errors_diff = """
+Line 19: Expected 1 errors
+Line 20: Expected 1 errors
+Line 38: Expected 1 errors
+Line 39: Expected 1 errors
+Line 50: Expected 1 errors
+Line 51: Expected 1 errors
+Line 52: Expected 1 errors
+Line 63: Expected 1 errors
+Line 69: Expected 1 errors
+Line 72: Expected 1 errors
+Line 75: Expected 1 errors
+"""
+output = """
+"""

conformance/results/ty/aliases_recursive.toml

@@ -0,0 +1,45 @@
+conformance_automated = "Fail"
+conformant = "Partial"
+notes = """
+Does not reject use of the `type` statement inside functions.
+Does not reject circular definitions of type aliases.
+Does not reject redeclarations of type aliases.
+Does not support `type` statements generic over `TypeVarTuple`s.
+"""
+errors_diff = """
+Line 56: Expected 1 errors
+Line 84: Expected 1 errors
+Lines 51, 52: Expected error (tag 'TA14')
+Line 10: Unexpected errors ['aliases_type_statement.py:10:52: error[invalid-type-arguments] Too many type arguments: expected 2, got 3', 'aliases_type_statement.py:10:52: error[invalid-type-form] `...` is not allowed in this context in a type expression']
+"""
+output = """
+aliases_type_statement.py:10:52: error[invalid-type-arguments] Too many type arguments: expected 2, got 3
+aliases_type_statement.py:10:52: error[invalid-type-form] `...` is not allowed in this context in a type expression
+aliases_type_statement.py:17:1: error[unresolved-attribute] Object of type `TypeAliasType` has no attribute `bit_count`
+aliases_type_statement.py:19:1: error[call-non-callable] Object of type `TypeAliasType` is not callable
+aliases_type_statement.py:23:7: error[unresolved-attribute] Object of type `TypeAliasType` has no attribute `other_attrib`
+aliases_type_statement.py:26:18: error[invalid-base] Invalid class base with type `TypeAliasType`
+aliases_type_statement.py:31:22: error[invalid-argument-type] Argument to function `isinstance` is incorrect: Expected `type | UnionType | tuple[Divergent, ...]`, found `TypeAliasType`
+aliases_type_statement.py:37:22: error[invalid-type-form] Function calls are not allowed in type expressions
+aliases_type_statement.py:38:22: error[invalid-type-form] List literals are not allowed in this context in a type expression: Did you mean `tuple[int, str]`?
+aliases_type_statement.py:39:22: error[invalid-type-form] Tuple literals are not allowed in this context in a type expression
+aliases_type_statement.py:39:23: error[invalid-type-form] Tuple literals are not allowed in this context in a type expression: Did you mean `tuple[int, str]`?
+aliases_type_statement.py:40:22: error[invalid-type-form] List comprehensions are not allowed in type expressions
+aliases_type_statement.py:41:22: error[invalid-type-form] Dict literals are not allowed in type expressions
+aliases_type_statement.py:42:22: error[invalid-type-form] Function calls are not allowed in type expressions
+aliases_type_statement.py:43:22: error[invalid-type-form] Invalid subscript of object of type `list[<class 'int'>]` in type expression
+aliases_type_statement.py:43:28: error[invalid-type-form] Int literals are not allowed in this context in a type expression
+aliases_type_statement.py:44:22: error[invalid-type-form] `if` expressions are not allowed in type expressions
+aliases_type_statement.py:45:22: error[invalid-type-form] Variable of type `Literal[1]` is not allowed in a type expression
+aliases_type_statement.py:46:23: error[invalid-type-form] Boolean literals are not allowed in this context in a type expression
+aliases_type_statement.py:47:23: error[invalid-type-form] Int literals are not allowed in this context in a type expression
+aliases_type_statement.py:48:23: error[invalid-type-form] Boolean operations are not allowed in type expressions
+aliases_type_statement.py:49:23: error[fstring-type-annotation] Type expressions cannot use f-strings
+aliases_type_statement.py:62:23: error[unbound-type-variable] Type variable `V` is not bound to any outer generic context
+aliases_type_statement.py:67:17: error[unbound-type-variable] Type variable `T1` is not bound to any outer generic context
+aliases_type_statement.py:77:27: error[invalid-type-arguments] Type `str` is not assignable to upper bound `int` of type variable `S@RecursiveTypeAlias2`
+aliases_type_statement.py:79:32: error[invalid-type-arguments] Type `int` is not assignable to upper bound `str` of type variable `T@RecursiveTypeAlias2`
+aliases_type_statement.py:82:1: error[cyclic-type-alias-definition] Cyclic definition of `RecursiveTypeAlias3`
+aliases_type_statement.py:88:1: error[cyclic-type-alias-definition] Cyclic definition of `RecursiveTypeAlias6`
+aliases_type_statement.py:89:1: error[cyclic-type-alias-definition] Cyclic definition of `RecursiveTypeAlias7`
+"""

conformance/results/ty/aliases_type_statement.toml

@@ -0,0 +1,36 @@
+conformance_automated = "Fail"
+conformant = "Partial"
+notes = """
+Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.
+Does not reject declaring a type alias with a type variable that is not in scope.
+Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.
+Does not reject cyclically defined type aliases.
+"""
+errors_diff = """
+Line 40: Expected 1 errors
+Line 43: Expected 1 errors
+Line 44: Expected 1 errors
+Line 45: Expected 1 errors
+Line 46: Expected 1 errors
+Line 47: Expected 1 errors
+Line 48: Expected 1 errors
+"""
+output = """
+aliases_typealiastype.py:32:7: error[unresolved-attribute] Object of type `TypeAliasType` has no attribute `other_attrib`
+aliases_typealiastype.py:52:40: error[invalid-type-form] Function calls are not allowed in type expressions
+aliases_typealiastype.py:53:40: error[invalid-type-form] List literals are not allowed in this context in a type expression: Did you mean `tuple[int, str]`?
+aliases_typealiastype.py:54:42: error[invalid-type-form] Tuple literals are not allowed in this context in a type expression
+aliases_typealiastype.py:54:43: error[invalid-type-form] Tuple literals are not allowed in this context in a type expression: Did you mean `tuple[int, str]`?
+aliases_typealiastype.py:55:42: error[invalid-type-form] List comprehensions are not allowed in type expressions
+aliases_typealiastype.py:56:42: error[invalid-type-form] Dict literals are not allowed in type expressions
+aliases_typealiastype.py:57:42: error[invalid-type-form] Function calls are not allowed in type expressions
+aliases_typealiastype.py:58:42: error[invalid-type-form] Invalid subscript of object of type `list[<class 'int'>]` in type expression
+aliases_typealiastype.py:58:48: error[invalid-type-form] Int literals are not allowed in this context in a type expression
+aliases_typealiastype.py:59:42: error[invalid-type-form] `if` expressions are not allowed in type expressions
+aliases_typealiastype.py:60:42: error[invalid-type-form] Variable of type `Literal[3]` is not allowed in a type expression
+aliases_typealiastype.py:61:42: error[invalid-type-form] Boolean literals are not allowed in this context in a type expression
+aliases_typealiastype.py:62:42: error[invalid-type-form] Int literals are not allowed in this context in a type expression
+aliases_typealiastype.py:63:42: error[invalid-type-form] Boolean operations are not allowed in type expressions
+aliases_typealiastype.py:64:42: error[invalid-type-form] F-strings are not allowed in type expressions
+aliases_typealiastype.py:66:47: error[unresolved-reference] Name `BadAlias21` used when not defined
+"""

conformance/results/ty/aliases_typealiastype.toml

@@ -0,0 +1,10 @@
+conformance_automated = "Fail"
+conformant = "Unsupported"
+errors_diff = """
+Line 24: Expected 1 errors
+Line 28: Expected 1 errors
+Line 32: Expected 1 errors
+Line 44: Expected 1 errors
+"""
+output = """
+"""