Merge branch 'master' into patch-11

Author: BobTheBuidler

mypy-requirements.txt

@@ -4,4 +4,4 @@ typing_extensions>=4.6.0
 mypy_extensions>=1.0.0
 pathspec>=0.9.0
 tomli>=1.1.0; python_version<'3.11'
-librt>=0.6.2
+librt>=0.6.2; platform_python_implementation != 'PyPy'

mypy/build.py

@@ -43,7 +43,14 @@
 from librt.internal import cache_version
 
 import mypy.semanal_main
-from mypy.cache import CACHE_VERSION, CacheMeta, ReadBuffer, WriteBuffer, write_json
+from mypy.cache import (
+    CACHE_VERSION,
+    CacheMeta,
+    ReadBuffer,
+    SerializedError,
+    WriteBuffer,
+    write_json,
+)
 from mypy.checker import TypeChecker
 from mypy.defaults import (
     WORKER_CONNECTION_TIMEOUT,
@@ -52,7 +59,7 @@
     WORKER_START_TIMEOUT,
 )
 from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter
-from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error
+from mypy.errors import CompileError, ErrorInfo, Errors, ErrorTuple, report_internal_error
 from mypy.graph_utils import prepare_sccs, strongly_connected_components, topsort
 from mypy.indirection import TypeIndirectionVisitor
 from mypy.ipc import BadStatus, IPCClient, read_status, ready_to_read, receive, send
@@ -2046,7 +2053,7 @@ class State:
     dep_hashes: dict[str, bytes] = {}
 
     # List of errors reported for this file last time.
-    error_lines: list[str] = []
+    error_lines: list[SerializedError] = []
 
     # Parent package, its parent, etc.
     ancestors: list[str] | None = None
@@ -3511,9 +3518,13 @@ def find_stale_sccs(
                 scc = order_ascc_ex(graph, ascc)
             for id in scc:
                 if graph[id].error_lines:
-                    manager.flush_errors(
-                        manager.errors.simplify_path(graph[id].xpath), graph[id].error_lines, False
+                    path = manager.errors.simplify_path(graph[id].xpath)
+                    formatted = manager.errors.format_messages(
+                        path,
+                        deserialize_codes(graph[id].error_lines),
+                        formatter=manager.error_formatter,
                     )
+                    manager.flush_errors(path, formatted, False)
             fresh_sccs.append(ascc)
         else:
             size = len(ascc.mod_ids)
@@ -3759,21 +3770,24 @@ def process_stale_scc(
     # Flush errors, and write cache in two phases: first data files, then meta files.
     meta_tuples = {}
     errors_by_id = {}
+    formatted_by_id = {}
     for id in stale:
         if graph[id].xpath not in manager.errors.ignored_files:
-            errors = manager.errors.file_messages(
-                graph[id].xpath, formatter=manager.error_formatter
+            errors = manager.errors.file_messages(graph[id].xpath)
+            formatted = manager.errors.format_messages(
+                graph[id].xpath, errors, formatter=manager.error_formatter
             )
-            manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False)
+            manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), formatted, False)
             errors_by_id[id] = errors
+            formatted_by_id[id] = formatted
         meta_tuples[id] = graph[id].write_cache()
     for id in stale:
         meta_tuple = meta_tuples[id]
         if meta_tuple is None:
             continue
         meta, meta_file = meta_tuple
         meta.dep_hashes = [graph[dep].interface_hash for dep in graph[id].dependencies]
-        meta.error_lines = errors_by_id.get(id, [])
+        meta.error_lines = serialize_codes(errors_by_id.get(id, []))
         write_cache_meta(meta, manager, meta_file)
     manager.done_sccs.add(ascc.id)
     manager.add_stats(
@@ -3785,7 +3799,7 @@ def process_stale_scc(
     )
     scc_result = {}
     for id in scc:
-        scc_result[id] = graph[id].interface_hash.hex(), errors_by_id.get(id, [])
+        scc_result[id] = graph[id].interface_hash.hex(), formatted_by_id.get(id, [])
     return scc_result
 
 
@@ -3932,3 +3946,26 @@ def sccs_to_bytes(sccs: list[SCC]) -> bytes:
     buf = WriteBuffer()
     write_json(buf, {"sccs": scc_tuples})
     return buf.getvalue()
+
+
+def serialize_codes(errs: list[ErrorTuple]) -> list[SerializedError]:
+    return [
+        (path, line, column, end_line, end_column, severity, message, code.code if code else None)
+        for path, line, column, end_line, end_column, severity, message, code in errs
+    ]
+
+
+def deserialize_codes(errs: list[SerializedError]) -> list[ErrorTuple]:
+    return [
+        (
+            path,
+            line,
+            column,
+            end_line,
+            end_column,
+            severity,
+            message,
+            codes.error_codes.get(code) if code else None,
+        )
+        for path, line, column, end_line, end_column, severity, message, code in errs
+    ]

mypy/cache.py

@@ -69,7 +69,9 @@
 from mypy_extensions import u8
 
 # High-level cache layout format
-CACHE_VERSION: Final = 0
+CACHE_VERSION: Final = 1
+
+SerializedError: _TypeAlias = tuple[str | None, int, int, int, int, str, str, str | None]
 
 
 class CacheMeta:
@@ -92,7 +94,7 @@ def __init__(
         dep_lines: list[int],
         dep_hashes: list[bytes],
         interface_hash: bytes,
-        error_lines: list[str],
+        error_lines: list[SerializedError],
         version_id: str,
         ignore_all: bool,
         plugin_data: Any,
@@ -157,7 +159,7 @@ def deserialize(cls, meta: dict[str, Any], data_file: str) -> CacheMeta | None:
                 dep_lines=meta["dep_lines"],
                 dep_hashes=[bytes.fromhex(dep) for dep in meta["dep_hashes"]],
                 interface_hash=bytes.fromhex(meta["interface_hash"]),
-                error_lines=meta["error_lines"],
+                error_lines=[tuple(err) for err in meta["error_lines"]],
                 version_id=meta["version_id"],
                 ignore_all=meta["ignore_all"],
                 plugin_data=meta["plugin_data"],
@@ -179,7 +181,7 @@ def write(self, data: WriteBuffer) -> None:
         write_int_list(data, self.dep_lines)
         write_bytes_list(data, self.dep_hashes)
         write_bytes(data, self.interface_hash)
-        write_str_list(data, self.error_lines)
+        write_errors(data, self.error_lines)
         write_str(data, self.version_id)
         write_bool(data, self.ignore_all)
         # Plugin data may be not a dictionary, so we use
@@ -204,7 +206,7 @@ def read(cls, data: ReadBuffer, data_file: str) -> CacheMeta | None:
                 dep_lines=read_int_list(data),
                 dep_hashes=read_bytes_list(data),
                 interface_hash=read_bytes(data),
-                error_lines=read_str_list(data),
+                error_lines=read_errors(data),
                 version_id=read_str(data),
                 ignore_all=read_bool(data),
                 plugin_data=read_json_value(data),
@@ -231,6 +233,7 @@ def read(cls, data: ReadBuffer, data_file: str) -> CacheMeta | None:
 LIST_INT: Final[Tag] = 21
 LIST_STR: Final[Tag] = 22
 LIST_BYTES: Final[Tag] = 23
+TUPLE_GEN: Final[Tag] = 24
 DICT_STR_GEN: Final[Tag] = 30
 
 # Misc classes.
@@ -391,12 +394,11 @@ def write_str_opt_list(data: WriteBuffer, value: list[str | None]) -> None:
 
 
 Value: _TypeAlias = None | int | str | bool
-JsonValue: _TypeAlias = Value | list["JsonValue"] | dict[str, "JsonValue"]
 
-# Currently tuples are used by mypyc plugin. They will be normalized to
-# JSON lists after a roundtrip.
-JsonValueEx: _TypeAlias = (
-    Value | list["JsonValueEx"] | dict[str, "JsonValueEx"] | tuple["JsonValueEx", ...]
+# Our JSON format is somewhat non-standard as we distinguish lists and tuples.
+# This is convenient for some internal things, like mypyc plugin and error serialization.
+JsonValue: _TypeAlias = (
+    Value | list["JsonValue"] | dict[str, "JsonValue"] | tuple["JsonValue", ...]
 )
 
 
@@ -415,13 +417,16 @@ def read_json_value(data: ReadBuffer) -> JsonValue:
     if tag == LIST_GEN:
         size = read_int_bare(data)
         return [read_json_value(data) for _ in range(size)]
+    if tag == TUPLE_GEN:
+        size = read_int_bare(data)
+        return tuple(read_json_value(data) for _ in range(size))
     if tag == DICT_STR_GEN:
         size = read_int_bare(data)
         return {read_str_bare(data): read_json_value(data) for _ in range(size)}
     assert False, f"Invalid JSON tag: {tag}"
 
 
-def write_json_value(data: WriteBuffer, value: JsonValueEx) -> None:
+def write_json_value(data: WriteBuffer, value: JsonValue) -> None:
     if value is None:
         write_tag(data, LITERAL_NONE)
     elif isinstance(value, bool):
@@ -432,11 +437,16 @@ def write_json_value(data: WriteBuffer, value: JsonValueEx) -> None:
     elif isinstance(value, str):
         write_tag(data, LITERAL_STR)
         write_str_bare(data, value)
-    elif isinstance(value, (list, tuple)):
+    elif isinstance(value, list):
         write_tag(data, LIST_GEN)
         write_int_bare(data, len(value))
         for val in value:
             write_json_value(data, val)
+    elif isinstance(value, tuple):
+        write_tag(data, TUPLE_GEN)
+        write_int_bare(data, len(value))
+        for val in value:
+            write_json_value(data, val)
     elif isinstance(value, dict):
         write_tag(data, DICT_STR_GEN)
         write_int_bare(data, len(value))
@@ -461,3 +471,38 @@ def write_json(data: WriteBuffer, value: dict[str, Any]) -> None:
     for key in sorted(value):
         write_str_bare(data, key)
         write_json_value(data, value[key])
+
+
+def write_errors(data: WriteBuffer, errs: list[SerializedError]) -> None:
+    write_tag(data, LIST_GEN)
+    write_int_bare(data, len(errs))
+    for path, line, column, end_line, end_column, severity, message, code in errs:
+        write_tag(data, TUPLE_GEN)
+        write_str_opt(data, path)
+        write_int(data, line)
+        write_int(data, column)
+        write_int(data, end_line)
+        write_int(data, end_column)
+        write_str(data, severity)
+        write_str(data, message)
+        write_str_opt(data, code)
+
+
+def read_errors(data: ReadBuffer) -> list[SerializedError]:
+    assert read_tag(data) == LIST_GEN
+    result = []
+    for _ in range(read_int_bare(data)):
+        assert read_tag(data) == TUPLE_GEN
+        result.append(
+            (
+                read_str_opt(data),
+                read_int(data),
+                read_int(data),
+                read_int(data),
+                read_int(data),
+                read_str(data),
+                read_str(data),
+                read_str_opt(data),
+            )
+        )
+    return result

mypy/checker.py

@@ -4116,8 +4116,19 @@ def check_multi_assignment(
                 lvalues, rvalue, rvalue_type, context, infer_lvalue_type
             )
         else:
+            rvalue_literal = rvalue_type
             if isinstance(rvalue_type, Instance) and rvalue_type.type.fullname == "builtins.str":
+                if rvalue_type.last_known_value is None:
+                    self.msg.unpacking_strings_disallowed(context)
+                else:
+                    rvalue_literal = rvalue_type.last_known_value
+            if (
+                isinstance(rvalue_literal, LiteralType)
+                and isinstance(rvalue_literal.value, str)
+                and len(lvalues) != len(rvalue_literal.value)
+            ):
                 self.msg.unpacking_strings_disallowed(context)
+
             self.check_multi_assignment_from_iterable(
                 lvalues, rvalue_type, context, infer_lvalue_type
             )
@@ -4363,9 +4374,7 @@ def check_multi_assignment_from_iterable(
         infer_lvalue_type: bool = True,
     ) -> None:
         rvalue_type = get_proper_type(rvalue_type)
-        if self.type_is_iterable(rvalue_type) and isinstance(
-            rvalue_type, (Instance, CallableType, TypeType, Overloaded)
-        ):
+        if self.type_is_iterable(rvalue_type):
             item_type = self.iterable_item_type(rvalue_type, context)
             for lv in lvalues:
                 if isinstance(lv, StarExpr):
@@ -7803,9 +7812,7 @@ def note(
             return
         self.msg.note(msg, context, offset=offset, code=code)
 
-    def iterable_item_type(
-        self, it: Instance | CallableType | TypeType | Overloaded, context: Context
-    ) -> Type:
+    def iterable_item_type(self, it: ProperType, context: Context) -> Type:
         if isinstance(it, Instance):
             iterable = map_instance_to_supertype(it, self.lookup_typeinfo("typing.Iterable"))
             item_type = iterable.args[0]
@@ -8744,8 +8751,9 @@ def is_unsafe_overlapping_overload_signatures(
     # Note: We repeat this check twice in both directions compensate for slight
     # asymmetries in 'is_callable_compatible'.
 
+    other_expanded = expand_callable_variants(other)
     for sig_variant in expand_callable_variants(signature):
-        for other_variant in expand_callable_variants(other):
+        for other_variant in other_expanded:
             # Using only expanded callables may cause false negatives, we can add
             # more variants (e.g. using inference between callables) in the future.
             if is_subset_no_promote(sig_variant.ret_type, other_variant.ret_type):

mypy/checkpattern.py

@@ -46,6 +46,7 @@
     Type,
     TypedDictType,
     TypeOfAny,
+    TypeType,
     TypeVarTupleType,
     TypeVarType,
     UninhabitedType,
@@ -556,6 +557,8 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType:
             fallback = self.chk.named_type("builtins.function")
             any_type = AnyType(TypeOfAny.unannotated)
             typ = callable_with_ellipsis(any_type, ret_type=any_type, fallback=fallback)
+        elif isinstance(p_typ, TypeType) and isinstance(p_typ.item, NoneType):
+            typ = p_typ.item
         elif not isinstance(p_typ, AnyType):
             self.msg.fail(
                 message_registry.CLASS_PATTERN_TYPE_REQUIRED.format(