fix(crossrepo): capture std::/boost:: callees for cross-lib linking (baselib_callees); OFF byte-identical

extract_callees previously dropped every std::/boost:: callee via SYSTEM_PREFIXES
at parse time, so collect_transitive_deps never saw them and the cross-repo
base-lib linker was DEAD for boost/std (the two highest-value libs). Capture
those callees in a separate FunctionDef.baselib_callees list (same parse walk),
consulted by the cross-lib linker only when --global-symbol-index is given.
Normal callees/call-edges and the flag-OFF enriched doc are byte-identical to
before (verified vs HEAD). Bounds unchanged: depth-1, cap 12/doc, 6144 tok/doc,
tagged crosslib:<repo>. Update process_commits.py for the new 2-tuple return.
Add real-libclang-parse regression tests (the unit tests hand-built callees and
bypassed the SYSTEM_PREFIXES filter, which is why the bug was hidden).

Author: apstenku123

tests/test_crossrepo_global_symbols.py

@@ -148,3 +148,108 @@ def test_crosslink_on_pulls_tagged_chunk(tmp_path):
         f"expected root->crosslib call_edge; edges={edges} root={root_idx} cl={cl_idx}"
     )
     reader.close()
+
+
+# --------------------------------------------------------------------------- #
+# REAL libclang parse-path regression. The unit tests above hand-build
+# FunctionDef.callees with a 'boost::' qname, which bypasses extract_callees and
+# its SYSTEM_PREFIXES filter. In the LIVE pipeline, base-lib callees go through
+# extract_callees first -- and 'std::'/'boost::' are in SYSTEM_PREFIXES, so they
+# were silently dropped from `callees` and never reached the cross-lib linker
+# (the feature was dead for boost/std). This test parses real source with
+# libclang to prove the cross-linkable base-lib callee survives into
+# FunctionDef.baselib_callees and is pulled when the index is enabled.
+# --------------------------------------------------------------------------- #
+def _parse_one(ip, src_text, tmp_path):
+    ip._configure_libclang()
+    from clang.cindex import Index  # type: ignore
+    src = tmp_path / "u.cpp"
+    src.write_text(src_text)
+    idx = Index.create()
+    funcs, _types = ip.parse_translation_unit(
+        str(src), idx, ["-std=c++17"], str(tmp_path))
+    return funcs
+
+
+def test_extract_callees_splits_baselib_from_normal():
+    ip = _load_index_project()
+    import clang.cindex  # noqa: F401  (skip cleanly if libclang missing)
+    out = ip.extract_callees  # signature check: returns a 2-tuple
+    assert ip.is_crosslinkable_baselib("boost::beast::make_printable")
+    assert ip.is_crosslinkable_baselib("std::sort")
+    assert not ip.is_crosslinkable_baselib("myrepo::helper")
+    assert not ip.is_crosslinkable_baselib("memcpy")
+    assert callable(out)
+
+
+def test_real_parse_baselib_callee_survives_and_pulls(tmp_path):
+    ip = _load_index_project()
+    try:
+        import clang.cindex  # noqa: F401
+    except Exception as exc:  # pragma: no cover
+        pytest.skip(f"libclang unavailable: {exc}")
+
+    # Root calls a boost:: symbol (filtered out of `callees`) + a local helper.
+    src = (
+        "namespace boost { namespace algorithm { template<class R> "
+        "void trim(R& r); } }\n"
+        "static int local_helper(int x) { return x + 1; }\n"
+        "int do_work(int s) {\n"
+        "    boost::algorithm::trim(s);\n"
+        "    return local_helper(s);\n"
+        "}\n"
+    )
+    funcs = _parse_one(ip, src, tmp_path)
+    root = next(f for f in funcs if f.name == "do_work")
+
+    # REGRESSION CORE: boost:: callee is NOT in `callees` (SYSTEM_PREFIXES) but
+    # IS captured in baselib_callees so the cross-lib linker can see it.
+    assert not any(c.startswith("boost::") for c in root.callees), root.callees
+    assert "boost::algorithm::trim" in root.baselib_callees, root.baselib_callees
+    # IPC round-trip preserves the new field.
+    assert ip.FunctionDef.from_dict(root.to_dict()).baselib_callees == \
+        root.baselib_callees
+
+    # Now build a store with the boost def and confirm an actual PULL.
+    db, body = _make_store(tmp_path)
+    reader = ip.GlobalSymbolReader(db)
+    idx = ip.ProjectIndex()
+    for f in funcs:
+        idx.add_function(f)
+    docs = ip.build_training_documents(
+        idx, max_tokens=16384, enriched=True, global_symbols=reader)
+    root_doc = [d for d in docs if "do_work" in d["text"]][0]
+    cl = [b for b in root_doc["chunk_boundaries"]
+          if (b.get("dep_source") or "").startswith("crosslib:")]
+    assert len(cl) == 1, cl
+    assert cl[0]["dep_source"] == "crosslib:boost"
+    assert cl[0]["name"] == "trim"
+    assert "trim impl" in root_doc["text"]
+    reader.close()
+
+
+def test_real_parse_off_has_no_crosslink(tmp_path):
+    """With global_symbols=None the base-lib callee is captured but never pulled
+    (OFF behavior unchanged: no crosslib chunk, no provenance)."""
+    ip = _load_index_project()
+    try:
+        import clang.cindex  # noqa: F401
+    except Exception as exc:  # pragma: no cover
+        pytest.skip(f"libclang unavailable: {exc}")
+    src = (
+        "namespace boost { namespace algorithm { template<class R> "
+        "void trim(R& r); } }\n"
+        "int do_work(int seed_value_for_token_floor) {\n"
+        "    int accumulator = seed_value_for_token_floor + 1;\n"
+        "    boost::algorithm::trim(accumulator);\n"
+        "    return accumulator + seed_value_for_token_floor;\n"
+        "}\n"
+    )
+    funcs = _parse_one(ip, src, tmp_path)
+    idx = ip.ProjectIndex()
+    for f in funcs:
+        idx.add_function(f)
+    docs = ip.build_training_documents(idx, max_tokens=16384, enriched=True)
+    root_doc = [d for d in docs if "do_work" in d["text"]][0]
+    assert not any(b.get("dep_source") for b in root_doc["chunk_boundaries"])
+    assert "trim impl" not in root_doc["text"]

tools/clang_indexer/index_project.py

@@ -348,7 +348,10 @@ def classify_build_file(fname: str) -> str | None:
     return None
 
 
-# System/stdlib function prefixes (skip for dependency tracking)
+# System/stdlib function prefixes (skip for dependency tracking). UNCHANGED from
+# the original: callees under these prefixes are NOT recorded as normal callees,
+# so the local dep-resolution / call-edge path behaves EXACTLY as before whether
+# or not cross-repo linking is enabled. (OFF behavior is byte-identical.)
 SYSTEM_PREFIXES = (
     'std::', 'boost::', '__builtin', '__', 'operator', 'printf', 'fprintf',
     'sprintf', 'snprintf', 'scanf', 'malloc', 'calloc', 'realloc', 'free',
@@ -357,19 +360,30 @@ def classify_build_file(fname: str) -> str | None:
     'assert', 'pthread_', 'EXPECT_', 'ASSERT_', 'TEST',
 )
 
+# Base-library NAMESPACE prefixes that ARE cross-linkable (present in the global
+# symbol store built by scripts/crossrepo/build_global_symbol_index.py). A callee
+# under one of these is filtered OUT of the normal `callees` list (above) but is
+# captured SEPARATELY in FunctionDef.baselib_callees so the cross-repo linker can
+# resolve it WITHOUT changing the normal callee/call-edge behavior. We do NOT
+# include libc-style bare names here (memcpy/strlen/...) because those are not in
+# the A1 selection and would only add noise; only the namespaced base libs.
+CROSSLINKABLE_NS_PREFIXES = ('std::', 'boost::')
+
 
 class FunctionDef:
     """A function definition with its source location and call references."""
     __slots__ = ['name', 'qualified_name', 'file', 'line', 'end_line', 'text',
                  'callees', 'referenced_types', 'dep_level', 'is_definition',
-                 'ast_depth', 'sibling_index', 'ast_node_type']
+                 'ast_depth', 'sibling_index', 'ast_node_type',
+                 'baselib_callees']
 
     def __init__(self, name: str, qualified_name: str, file: str, line: int,
                  text: str, callees: list, is_definition: bool = True,
                  end_line: int = 0, ast_depth: list[int] | None = None,
                  sibling_index: list[int] | None = None,
                  ast_node_type: list[int] | None = None,
-                 referenced_types: list | None = None):
+                 referenced_types: list | None = None,
+                 baselib_callees: list | None = None):
         self.name = name
         self.qualified_name = qualified_name
         self.file = file
@@ -382,6 +396,10 @@ def __init__(self, name: str, qualified_name: str, file: str, line: int,
         # SAME libclang parse as callees so it round-trips IPC and feeds the
         # offline type_refs/type_edges builders. Mirrors `callees`.
         self.referenced_types = list(referenced_types or [])
+        # Cross-linkable base-lib callees (std::/boost::) dropped from `callees`
+        # by the normal system-prefix filter, kept SEPARATELY for the optional
+        # cross-repo linker. Empty/ignored unless --global-symbol-index is given.
+        self.baselib_callees = list(baselib_callees or [])
         self.dep_level = 0
         self.is_definition = is_definition
         self.ast_depth = list(ast_depth or [])
@@ -399,6 +417,7 @@ def to_dict(self) -> dict:
             'sibling_index': self.sibling_index,
             'ast_node_type': self.ast_node_type,
             'referenced_types': self.referenced_types,
+            'baselib_callees': self.baselib_callees,
         }
 
     @classmethod
@@ -537,6 +556,19 @@ def is_system_function(name: str | None) -> bool:
     return any(name.startswith(p) for p in SYSTEM_PREFIXES)
 
 
+def is_crosslinkable_baselib(name: str | None) -> bool:
+    """True if ``name`` is a cross-linkable base-lib symbol (std::/boost::).
+
+    Such a callee is filtered out of the normal ``callees`` list (it matches
+    SYSTEM_PREFIXES) but is captured separately so the cross-repo linker can
+    resolve it against the global symbol index. This does NOT affect normal
+    callee/call-edge behavior.
+    """
+    if not isinstance(name, str) or not name:
+        return False
+    return any(name.startswith(p) for p in CROSSLINKABLE_NS_PREFIXES)
+
+
 _DROP_CLANG_ARG_PREFIXES = (
     "-fsanitize=",
     "-fno-sanitize=",
@@ -813,23 +845,39 @@ def _cursor_text_and_metadata(
     )
 
 
-def extract_callees(cursor: Cursor) -> list[str]:
-    """Extract all function call references from a cursor's children."""
-    callees = set()
+def extract_callees(cursor: Cursor) -> tuple[list[str], list[str]]:
+    """Extract function call references from a cursor's children.
+
+    Returns ``(callees, baselib_callees)``:
+
+    * ``callees`` — normal resolvable callees (UNCHANGED filter: system/stdlib
+      prefixes dropped). Feeds local dep-resolution + call-edges exactly as
+      before, so OFF behavior is byte-identical.
+    * ``baselib_callees`` — callees that were dropped by the normal filter BUT
+      are cross-linkable base-lib namespaces (std::/boost::). Captured here in the
+      SAME walk so the cross-repo linker can resolve them against the global
+      symbol index WITHOUT touching the normal callee path. Ignored entirely when
+      cross-repo linking is disabled.
+    """
+    callees: set[str] = set()
+    baselib_callees: set[str] = set()
 
     def walk(node: Cursor):
         if node.kind == CursorKind.CALL_EXPR:
             ref = node.referenced
             if ref and ref.spelling:
                 # Get fully qualified name
                 qname = get_qualified_name(ref)
-                if qname and not is_system_function(qname):
-                    callees.add(qname)
+                if qname:
+                    if not is_system_function(qname):
+                        callees.add(qname)
+                    elif is_crosslinkable_baselib(qname):
+                        baselib_callees.add(qname)
         for child in node.get_children():
             walk(child)
 
     walk(cursor)
-    return list(callees)
+    return list(callees), list(baselib_callees)
 
 
 _REFERENCED_TYPE_DECL_KINDS = frozenset({
@@ -1020,7 +1068,7 @@ def visit(cursor):
                 )
             )
             if text and len(text) >= 20:
-                callees = extract_callees(cursor)
+                callees, baselib_callees = extract_callees(cursor)
                 referenced_types = extract_referenced_types(cursor)
                 qname = get_qualified_name(cursor)
                 functions.append(FunctionDef(
@@ -1036,6 +1084,7 @@ def visit(cursor):
                     sibling_index=func_sibling_index,
                     ast_node_type=func_ast_node_type,
                     referenced_types=referenced_types,
+                    baselib_callees=baselib_callees,
                 ))
             return
 
@@ -1380,6 +1429,29 @@ def collect_transitive_deps(
     the base lib (depth-1 only), and we SKIP trivial/tiny inline bodies — that is
     bounded by ``crosslink_budget`` + the builder's body-len floor.
     """
+    crosslink_active = (
+        global_symbols is not None
+        and crosslink_visited is not None
+        and crosslink_budget is not None
+    )
+
+    def _try_crosslink(callee: str) -> None:
+        # Depth-1 only: a pulled base-lib def is a LEAF; we never enqueue it, so
+        # its base-lib-internal callees are not followed. Bounds (count + token
+        # budget) are enforced by crosslink_budget. ONE clear path; a miss (not a
+        # selected base-lib symbol) is simply a no-op, never a fallback.
+        if callee in crosslink_visited:
+            return
+        if not crosslink_budget.has_room():
+            return
+        hit = global_symbols.lookup(callee)
+        if hit is None:
+            return
+        if not crosslink_budget.can_afford(hit["token_est"]):
+            return
+        crosslink_visited.add(callee)
+        crosslink_budget.spend(hit["token_est"])
+
     visited = {root_qname}
     queue = deque([(root_qname, 0)])
     deps = []
@@ -1398,25 +1470,16 @@ def collect_transitive_deps(
                 visited.add(callee)
                 deps.append(callee)
                 queue.append((callee, depth + 1))
-            elif (
-                global_symbols is not None
-                and crosslink_visited is not None
-                and crosslink_budget is not None
-            ):
-                # Unresolved callee: try the cross-repo base-lib index. Depth-1
-                # only — a pulled base-lib def is a LEAF here; we never enqueue it
-                # so its base-lib-internal callees are not followed.
-                if callee in crosslink_visited:
-                    continue
-                if not crosslink_budget.has_room():
-                    continue
-                hit = global_symbols.lookup(callee)
-                if hit is None:
-                    continue
-                if not crosslink_budget.can_afford(hit["token_est"]):
-                    continue
-                crosslink_visited.add(callee)
-                crosslink_budget.spend(hit["token_est"])
+            elif crosslink_active:
+                # An unresolved callee (not local) MIGHT be a base-lib symbol
+                # (e.g. a vendored/forward-declared libc-style name). Try it.
+                _try_crosslink(callee)
+        # Base-lib-namespaced callees (std::/boost::) were filtered out of
+        # `callees` by the normal system-prefix filter and live here. They are
+        # the PRIMARY cross-link source. Only consulted when cross-linking is on.
+        if crosslink_active:
+            for callee in func.baselib_callees:
+                _try_crosslink(callee)
 
     return deps
 

tools/clang_indexer/process_commits.py

@@ -718,7 +718,7 @@ def visit(cursor):
                 )
             )
             if text and len(text) >= 20:
-                callees = extract_callees(cursor)
+                callees, baselib_callees = extract_callees(cursor)
                 referenced_types = extract_referenced_types(cursor)
                 qname = get_qualified_name(cursor)
                 start_line = cursor.extent.start.line
@@ -736,6 +736,7 @@ def visit(cursor):
                     sibling_index=func_sibling_index,
                     ast_node_type=func_ast_node_type,
                     referenced_types=referenced_types,
+                    baselib_callees=baselib_callees,
                 ))
 
         elif cursor.kind in (CursorKind.CLASS_DECL, CursorKind.STRUCT_DECL,