[mypyc] Backport cached-group Extension.depends fix from upstream

mypyc/build.py

@@ -597,8 +597,14 @@ def get_header_deps(cfiles: list[tuple[str, str]]) -> list[tuple[bool, str]]:
     the on-disk headers after every group has written its files.
 
     Arguments:
-        cfiles: A list of (file name, file contents) pairs.
+        cfiles: A list of (file name, file contents) pairs. Contents must be
+            non-empty; callers handling cached groups must re-read the .c
+            from disk before calling, otherwise direct includes are missed
+            and Extension.depends ends up empty.
     """
+    assert all(
+        contents for _, contents in cfiles
+    ), "get_header_deps requires non-empty file contents"
     headers: set[tuple[bool, str]] = set()
     for _, contents in cfiles:
         headers.update(_extract_includes(contents))
@@ -737,7 +743,18 @@ def mypyc_build(
                 write_file(cfile_full, ctext)
             if os.path.splitext(cfile_full)[1] == ".c":
                 cfilenames.append(cfile_full)
-            per_cfile_deps.append((cfile_full, get_header_deps([(cfile, ctext)])))
+            # For fully-cached groups ctext is empty; read the on-disk .c so the dep resolver
+            # can walk its transitive header chain and populate Extension.depends. Otherwise,
+            # cross-group export-table header changes (e.g. a new class shifting struct offsets)
+            # won't trigger a recompile of this cached consumer's .o.
+            if not ctext and os.path.exists(cfile_full):
+                try:
+                    with open(cfile_full, encoding="utf-8") as _f:
+                        ctext = _f.read()
+                except OSError:
+                    pass
+            if ctext:
+                per_cfile_deps.append((cfile_full, get_header_deps([(cfile, ctext)])))
 
         # Fully-cached mypy build (typical of pip's second setup.py invocation
         # for the wheel-build phase): mypyc returns an empty ctext for the
@@ -758,9 +775,10 @@ def mypyc_build(
                         existing_text = _f.read()
                 except OSError:
                     existing_text = ""
-                per_cfile_deps.append(
-                    (existing, get_header_deps([(os.path.basename(existing), existing_text)]))
-                )
+                if existing_text:
+                    per_cfile_deps.append(
+                        (existing, get_header_deps([(os.path.basename(existing), existing_text)]))
+                    )
 
         pending.append(per_cfile_deps)
         group_cfilenames.append((cfilenames, []))

mypyc/test-data/run-multimodule.test

@@ -1391,6 +1391,60 @@ def translate(b: bytes) -> bytes:
 import native
 assert native.translate(b'ABCD') == b'BBCD'
 
+[case testIncrementalCrossGroupExportTableOffsets]
+# Regression: under separate=True, each consumer module's IR is
+# compiled against the positional layout of its deps'
+# `exports_<group>` struct. Reordering the dep's classes keeps the
+# same set of public names, so mypy's interface hash for the dep is
+# unchanged -- the consumer is not invalidated and stays fully
+# cached, which causes `_load_cached_group_files` to return empty
+# cfile content for the consumer's group.
+#
+# Before the fix, `get_header_deps` over empty content returned no
+# includes, so `Extension.depends` for the consumer ended up empty
+# and setuptools never recompiled the consumer's .o when the dep's
+# `__native_<dep>.h` shifted struct offsets. The stale .o kept the
+# old offsets and silently resolved cross-group calls to the wrong
+# class.
+from other_classes import Gamma, Delta
+
+def make_gamma() -> Gamma:
+    return Gamma()
+
+def make_delta() -> Delta:
+    return Delta()
+
+[file other_classes.py]
+class Alpha:
+    a: int = 1
+
+class Beta:
+    b: int = 2
+
+class Gamma:
+    g: int = 3
+
+class Delta:
+    d: int = 4
+
+[file other_classes.py.2]
+class Delta:
+    d: int = 4
+
+class Alpha:
+    a: int = 1
+
+class Beta:
+    b: int = 2
+
+class Gamma:
+    g: int = 3
+
+[file driver.py]
+import native
+assert type(native.make_gamma()).__name__ == "Gamma"
+assert type(native.make_delta()).__name__ == "Delta"
+
 [case testCrossModuleAttrDefaults]
 from other import Parent
 

mypyc/test/testutil.py

@@ -232,8 +232,14 @@ def show_c(cfiles: list[list[tuple[str, str]]]) -> None:
 
 
 def fudge_dir_mtimes(dir: str, delta: int) -> None:
+    # Skip linker outputs. Pushing them back combines with write_file's
+    # +1 sec bump on .c files to make .c always newer than .so, forcing
+    # an unconditional rebuild that would mask Extension.depends bugs.
+    # See setuptools/_distutils/command/build_ext.py:`build_extension`.
     for dirpath, _, filenames in os.walk(dir):
         for name in filenames:
+            if name.endswith((".so", ".pyd", ".o", ".obj")):
+                continue
             path = os.path.join(dirpath, name)
             new_mtime = os.stat(path).st_mtime + delta
             os.utime(path, times=(new_mtime, new_mtime))