Fix crash on partially typed namespace package (#20742)

Fixes #16214
Closes #20658

The root cause is similar to the issue fixed by
#19044, when there are multiple
packages in `google` namespace, some with `py.typed`, some without. As a
result, there can be a situation where we first find the untyped one,
and mark ancestor (i.e. `google`) as suppressed, but then we find the
typed one and actually add ancestor (i.e. `google`) to the graph. This
creates a situation where we write some references to `google` in cache,
while it may not be loaded on the warm run, thus causing a crash.

My solution is simple: un-suppress namespace ancestors if there is at
least one package that needs them, so that the build graph is
consistent.

Unfortunately, this uncovers another (pre-existing) bug where `# type:
ignore`s are not handled correctly on warm runs, similar to
#20105. In any case, I am going to
submit a separate PR for this bug (I think there may be a simple fix for
it).

Author: ilevkivskyi

mypy/build.py

@@ -3144,9 +3144,9 @@ def find_module_and_diagnose(
             raise ModuleNotFound
         if is_silent_import_module(manager, result) and not root_source:
             follow_imports = "silent"
-        return (result, follow_imports)
+        return result, follow_imports
     else:
-        # Could not find a module.  Typically the reason is a
+        # Could not find a module.  Typically, the reason is a
         # misspelled module name, missing stub, module not in
         # search path or the module has not been installed.
 
@@ -3155,7 +3155,7 @@ def find_module_and_diagnose(
         # Don't honor a global (not per-module) ignore_missing_imports
         # setting for modules that used to have bundled stubs, as
         # otherwise updating mypy can silently result in new false
-        # negatives. (Unless there are stubs but they are incomplete.)
+        # negatives. (Unless there are stubs, but they are incomplete.)
         global_ignore_missing_imports = manager.options.ignore_missing_imports
         if (
             is_module_from_legacy_bundled_package(id)
@@ -3650,7 +3650,7 @@ def load_graph(
             added = [dep for dep in st.suppressed if find_module_simple(dep, manager)]
         else:
             # During initial loading we don't care about newly added modules,
-            # they will be taken care of during fine grained update. See also
+            # they will be taken care of during fine-grained update. See also
             # comment about this in `State.__init__()`.
             added = []
         for dep in st.ancestors + dependencies + st.suppressed:
@@ -3710,11 +3710,15 @@ def load_graph(
                     assert newst.id not in graph, newst.id
                     graph[newst.id] = newst
                     new.append(newst)
-            if dep in graph and dep in st.suppressed_set:
-                # Previously suppressed file is now visible
+    # There are two things we need to do after the initial load loop. One is up-suppress
+    # modules that are back in graph. We need to do this after the loop to cover an edge
+    # case where a namespace package ancestor is shared by a typed and an untyped package.
+    for st in graph.values():
+        for dep in st.suppressed:
+            if dep in graph:
                 st.add_dependency(dep)
-    # In the loop above we skip indirect dependencies, so to make indirect dependencies behave
-    # more consistently with regular ones, we suppress them manually here (when needed).
+    # Second, in the initial loop we skip indirect dependencies, so to make indirect dependencies
+    # behave more consistently with regular ones, we suppress them manually here (when needed).
     for st in graph.values():
         indirect = [dep for dep in st.dependencies if st.priorities.get(dep) == PRI_INDIRECT]
         for dep in indirect:

mypy/modulefinder.py

@@ -471,7 +471,7 @@ def _find_module(self, id: str, use_typeshed: bool) -> tuple[ModuleSearchResult,
                 if fscache.isdir(path):
                     if fscache.isfile(stub_typed_file):
                         # Stub packages can have a py.typed file, which must include
-                        # 'partial\n' to make the package partial
+                        # 'partial\n' to make the package partial.
                         # Partial here means that mypy should look at the runtime
                         # package if installed.
                         if fscache.read(stub_typed_file).decode().strip() == "partial":

test-data/unit/pep561.test

@@ -233,3 +233,21 @@ our/bar.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#
 [out2]
 our/bar.py:1: error: Skipping analyzing "typedpkg_ns.b": module is installed, but missing library stubs or py.typed marker
 our/bar.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+
+[case testPartiallyTypedNamespacePackageNoCrash]
+# pkgs: typedpkg_ns_nested
+import a
+[file a.py]
+import m1
+import m2
+[file a.py.2]
+import m1
+[file m1.py]
+import typedpkg_ns.b  # type: ignore
+[file m2.py]
+import typedpkg_ns.a
+# TODO: the error on second run is a bug, but it is a separate issue, likely #20105.
+[out]
+[out2]
+m1.py:1: error: Skipping analyzing "typedpkg_ns": module is installed, but missing library stubs or py.typed marker
+m1.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports