Merge remote-tracking branch 'upstream/master' into batch-sccs-parallel

Author: ilevkivskyi

CHANGELOG.md

@@ -2,11 +2,45 @@
 
 ## Next Release
 
+### Enabling `--local-partial-types` by default
+
+This flag affects the inference of types based on assignments in other scopes.
+For now, explicitly disabling this continues to be supported, but this support will be removed
+in the future as the legacy behaviour is hard to support with other current and future features
+in mypy, like the daemon or the new implementation of flexible redefinitions.
+
+Contributed by Ivan Levkivskyi, Jukka Lehtosalo, Shantanu in [PR 21163](https://github.com/python/mypy/pull/21163)
+
+### Enabling `--strict-bytes` by default
+
+Per [PEP 688](https://peps.python.org/pep-0688), mypy no longer treats `bytearray` and `memoryview`
+values as assignable to the `bytes` type.
+
+Contributed by Shantanu in [PR 18371](https://github.com/python/mypy/pull/18371)
+
 ### Drop Support for Targeting Python 3.9
 
 Mypy no longer supports type checking code with `--python-version 3.9`.
 Use `--python-version 3.10` or newer.
 
+Contributed by Shantanu, Marc Mueller in [PR 21243](https://github.com/python/mypy/pull/21243)
+
+### Remove special casing of legacy bundled stubs
+
+Mypy used to bundle stubs for a few packages in versions 0.812 and earlier. To navigate the
+transition, mypy used to report missing types for these packages even if `--ignore-missing-imports`
+was set. Mypy now consistently respects `--ignore-missing-imports` for all packages.
+
+Contributed by Shantanu in [PR 18372](https://github.com/python/mypy/pull/18372)
+
+### Prevent assignment to None for non-Optional class variables with type comments
+
+Mypy used to allow assignment to None for class variables when using type comments. This was a
+common idiom in Python 3.5 and earlier, prior to the introduction of variable annotations.
+However, this was a soundness hole and has now been removed.
+
+Contributed by Shantanu in [PR 20054](https://github.com/python/mypy/pull/20054)
+
 ## Mypy 1.20
 
 We’ve just uploaded mypy 1.20.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)).
@@ -544,6 +578,18 @@ Please see [git log](https://github.com/python/typeshed/commits/main?after=f8f07
 - Allow dangerous identity comparisons to `Any` typed variables (Shantanu, PR [21142](https://github.com/python/mypy/pull/21142))
 - `--warn-unused-config` should not be a strict flag (Ivan Levkivskyi, PR [21139](https://github.com/python/mypy/pull/21139))
 
+### Mypy 1.20.2
+
+- Use WAL with SQLite cache and fix close (Shantanu, PR [21154](https://github.com/python/mypy/pull/21154))
+- Adjust SQLite journal mode (Ivan Levkivskyi, PR [21217](https://github.com/python/mypy/pull/21217))
+- Correctly aggregate narrowing information on parent expressions (Shantanu, PR [21206](https://github.com/python/mypy/pull/21206))
+- Fix regression related to generic callables (Shantanu, PR [21208](https://github.com/python/mypy/pull/21208))
+- Fix regression by avoiding widening types in some contexts (Shantanu, PR [21242](https://github.com/python/mypy/pull/21242))
+- Fix slicing in non-strict optional mode (Shantanu, PR [21282](https://github.com/python/mypy/pull/21282))
+- mypyc: Fix match statement semantics for "or" pattern (Shantanu, PR [21156](https://github.com/python/mypy/pull/21156))
+- mypyc: Fix issue with module dunder attributes (Piotr Sawicki, PR [21275](https://github.com/python/mypy/pull/21275))
+- Initial support for Python 3.15.0a8 (Marc Mueller, PR [21255](https://github.com/python/mypy/pull/21255))
+
 ### Acknowledgements
 
 Thanks to all mypy contributors who contributed to this release:

misc/update-stubinfo.py

@@ -30,13 +30,10 @@ def main() -> None:
 
     import mypy.stubinfo
 
-    mypy_p = set(mypy.stubinfo.non_bundled_packages_flat) | set(
-        mypy.stubinfo.legacy_bundled_packages
-    )
+    mypy_p = set(mypy.stubinfo.non_bundled_packages_flat)
 
     for p in typeshed_p_to_d.keys() & mypy_p:
         mypy_d = mypy.stubinfo.non_bundled_packages_flat.get(p)
-        mypy_d = mypy_d or mypy.stubinfo.legacy_bundled_packages.get(p)
         if mypy_d != typeshed_p_to_d[p]:
             raise ValueError(
                 f"stub_distribution mismatch for {p}: {mypy_d} != {typeshed_p_to_d[p]}"

mypy/build.py

@@ -170,7 +170,7 @@
 from mypy.plugins.default import DefaultPlugin
 from mypy.renaming import LimitedVariableRenameVisitor, VariableRenameVisitor
 from mypy.stats import dump_type_stats
-from mypy.stubinfo import is_module_from_legacy_bundled_package, stub_distribution_name
+from mypy.stubinfo import stub_distribution_name
 from mypy.types import Type, instance_cache
 from mypy.typestate import reset_global_state, type_state
 from mypy.util import json_dumps, json_loads
@@ -955,7 +955,7 @@ def __init__(
         # until all the files have been added. This means that a
         # new file can be processed O(n**2) times. This cache
         # avoids most of this redundant work.
-        self.ast_cache: dict[str, tuple[MypyFile, list[ErrorInfo]]] = {}
+        self.ast_cache: dict[str, tuple[MypyFile, list[ErrorInfo], str | None]] = {}
         # Number of times we used GC optimization hack for fresh SCCs.
         self.gc_freeze_cycles = 0
         # Mapping from SCC id to corresponding SCC instance. This is populated
@@ -1043,11 +1043,68 @@ def parse_parallel(self, sequential_states: list[State], parallel_states: list[S
         as an optimization to parallelize only those parts of the code that can be
         parallelized efficiently.
         """
+        parallel_parsed_states, parallel_parsed_states_set = self.parse_files_threaded_raw(
+            sequential_states, parallel_states
+        )
+
+        for state in parallel_parsed_states:
+            # New parser returns serialized ASTs. Deserialize full trees only if not using
+            # parallel workers.
+            with state.wrap_context():
+                assert state.tree is not None
+                raw_data = state.tree.raw_data
+                if raw_data is not None:
+                    # Apply inline mypy config before deserialization, since
+                    # some options (e.g. implicit_optional) affect deserialization
+                    state.source_hash = raw_data.source_hash
+                    state.apply_inline_configuration(raw_data.mypy_comments)
+                    state.tree = load_from_raw(
+                        state.xpath,
+                        state.id,
+                        raw_data,
+                        self.errors,
+                        state.options,
+                        imports_only=bool(self.workers),
+                    )
+                if self.errors.is_blockers():
+                    self.log("Bailing due to parse errors")
+                    self.errors.raise_error()
+
+        for state in parallel_states:
+            assert state.tree is not None
+            if state in parallel_parsed_states_set:
+                if state.tree.raw_data is not None:
+                    # source_hash was already extracted above, but raw_data
+                    # may have been preserved for workers (imports_only=True).
+                    pass
+                elif state.source_hash is None:
+                    # At least namespace packages may not have source.
+                    state.get_source()
+                state.size_hint = os.path.getsize(state.xpath)
+                state.early_errors = list(self.errors.error_info_map.get(state.xpath, []))
+                state.semantic_analysis_pass1()
+                self.ast_cache[state.id] = (state.tree, state.early_errors, state.source_hash)
+            self.modules[state.id] = state.tree
+            if state.tree.raw_data is not None:
+                state.size_hint = len(state.tree.raw_data.defs) + MIN_SIZE_HINT
+            state.check_blockers()
+            state.setup_errors()
+
+    def parse_files_threaded_raw(
+        self, sequential_states: list[State], parallel_states: list[State]
+    ) -> tuple[list[State], set[State]]:
+        """Parse files using a thread pool.
+
+        Also parse sequential states while waiting for the parallel results.
+        Trees from the new parser are left in raw (serialized) form.
+
+        Return (list, set) of states that were actually parsed (not cached).
+        """
         futures = []
         # Use both list and a set to have more predictable order of errors,
         # while also not sacrificing performance.
-        parallel_parsed_states = []
-        parallel_parsed_states_set = set()
+        parallel_parsed_states: list[State] = []
+        parallel_parsed_states_set: set[State] = set()
         # Use at least --num-workers if specified by user.
         available_threads = max(get_available_threads(), self.options.num_workers)
         # Overhead from trying to parallelize (small) blocking portion of
@@ -1056,56 +1113,27 @@ def parse_parallel(self, sequential_states: list[State], parallel_states: list[S
         with ThreadPoolExecutor(max_workers=min(available_threads, 8)) as executor:
             for state in parallel_states:
                 state.needs_parse = False
-                # New parser reads source from file directly, we do this only for
-                # the side effect of parsing inline mypy configurations.
-                state.get_source()
                 if state.id not in self.ast_cache:
                     self.log(f"Parsing {state.xpath} ({state.id})")
                     ignore_errors = state.ignore_all or state.options.ignore_errors
                     if ignore_errors:
                         self.errors.ignored_files.add(state.xpath)
-                    futures.append(executor.submit(state.parse_file_inner, state.source or ""))
+                    futures.append(executor.submit(state.parse_file_inner, ""))
                     parallel_parsed_states.append(state)
                     parallel_parsed_states_set.add(state)
                 else:
                     self.log(f"Using cached AST for {state.xpath} ({state.id})")
-                    state.tree, state.early_errors = self.ast_cache[state.id]
+                    state.tree, state.early_errors, source_hash = self.ast_cache[state.id]
+                    state.source_hash = source_hash
 
             # Parse sequential before waiting on parallel.
             for state in sequential_states:
                 state.parse_file()
 
             for fut in wait(futures).done:
                 fut.result()
-            for state in parallel_parsed_states:
-                # New parser returns serialized trees that need to be de-serialized.
-                with state.wrap_context():
-                    assert state.tree is not None
-                    if state.tree.raw_data:
-                        state.tree = load_from_raw(
-                            state.xpath,
-                            state.id,
-                            state.tree.raw_data,
-                            self.errors,
-                            state.options,
-                            imports_only=bool(self.workers),
-                        )
-                    if self.errors.is_blockers():
-                        self.log("Bailing due to parse errors")
-                        self.errors.raise_error()
 
-        for state in parallel_states:
-            assert state.tree is not None
-            if state in parallel_parsed_states_set:
-                state.early_errors = list(self.errors.error_info_map.get(state.xpath, []))
-                state.semantic_analysis_pass1()
-                self.ast_cache[state.id] = (state.tree, state.early_errors)
-            self.modules[state.id] = state.tree
-            assert state.tree is not None
-            if state.tree.raw_data is not None:
-                state.size_hint = len(state.tree.raw_data.defs) + MIN_SIZE_HINT
-            state.check_blockers()
-            state.setup_errors()
+        return parallel_parsed_states, parallel_parsed_states_set
 
     def post_parse_all(self, states: list[State]) -> None:
         for state in states:
@@ -3134,7 +3162,6 @@ def get_source(self) -> str:
                 self.source_hash = compute_hash(source)
 
             self.parse_inline_configuration(source)
-            self.check_for_invalid_options()
 
             self.size_hint = len(source) + MIN_SIZE_HINT
         self.time_spent_us += time_spent_us(t0)
@@ -3159,7 +3186,10 @@ def parse_file(self, *, temporary: bool = False, raw_data: FileRawData | None =
             # The file was already parsed.
             return
 
-        source = self.get_source()
+        if raw_data is None:
+            source = self.get_source()
+        else:
+            source = ""
         manager = self.manager
         # Can we reuse a previously parsed AST? This avoids redundant work in daemon.
         if self.id not in manager.ast_cache:
@@ -3169,6 +3199,12 @@ def parse_file(self, *, temporary: bool = False, raw_data: FileRawData | None =
                 self.manager.errors.ignored_files.add(self.xpath)
             with self.wrap_context():
                 manager.errors.set_file(self.xpath, self.id, options=self.options)
+                if raw_data is not None:
+                    # Apply inline mypy config before deserialization, since
+                    # some options (e.g. implicit_optional) affect how the
+                    # AST is built during deserialization.
+                    self.source_hash = raw_data.source_hash
+                    self.apply_inline_configuration(raw_data.mypy_comments)
                 self.parse_file_inner(source, raw_data)
                 assert self.tree is not None
                 # New parser returns serialized trees that need to be de-serialized.
@@ -3193,14 +3229,15 @@ def parse_file(self, *, temporary: bool = False, raw_data: FileRawData | None =
         else:
             # Reuse a cached AST
             manager.log(f"Using cached AST for {self.xpath} ({self.id})")
-            self.tree, self.early_errors = manager.ast_cache[self.id]
+            self.tree, self.early_errors, source_hash = manager.ast_cache[self.id]
+            self.source_hash = source_hash
 
         assert self.tree is not None
         if not temporary:
             manager.modules[self.id] = self.tree
             self.check_blockers()
 
-        manager.ast_cache[self.id] = (self.tree, self.early_errors)
+        manager.ast_cache[self.id] = (self.tree, self.early_errors, self.source_hash)
         assert self.tree is not None
         if self.tree.raw_data is not None:
             # Size of serialized tree is a better proxy for file complexity than
@@ -3221,12 +3258,17 @@ def setup_errors(self) -> None:
     def parse_inline_configuration(self, source: str) -> None:
         """Check for inline mypy: options directive and parse them."""
         flags = get_mypy_comments(source)
+        self.apply_inline_configuration(flags)
+
+    def apply_inline_configuration(self, flags: list[tuple[int, str]] | None) -> None:
+        """Apply inline mypy configuration comments and check for invalid options."""
         if flags:
             changes, config_errors = parse_mypy_comments(flags, self.options)
             self.options = self.options.apply_changes(changes)
             self.manager.errors.set_file(self.xpath, self.id, self.options)
             for lineno, error in config_errors:
                 self.manager.error(lineno, error)
+        self.check_for_invalid_options()
 
     def check_for_invalid_options(self) -> None:
         if self.options.mypyc and not self.options.strict_bytes:
@@ -3714,19 +3756,6 @@ def find_module_and_diagnose(
 
         ignore_missing_imports = options.ignore_missing_imports
 
-        # Don't honor a global (not per-module) ignore_missing_imports
-        # setting for modules that used to have bundled stubs, as
-        # otherwise updating mypy can silently result in new false
-        # negatives. (Unless there are stubs, but they are incomplete.)
-        global_ignore_missing_imports = manager.options.ignore_missing_imports
-        if (
-            is_module_from_legacy_bundled_package(id)
-            and global_ignore_missing_imports
-            and not options.ignore_missing_imports_per_module
-            and result is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
-        ):
-            ignore_missing_imports = False
-
         if skip_diagnose:
             raise ModuleNotFound
         if caller_state: