Add full macro route regeneration pipeline

Author: apstenku123

cppmega_mlx/config/model.py

@@ -136,7 +136,7 @@ class StructureConfig:
 
     active_components: str = "core"
     bottleneck_dim: int = 64
-    num_categories: int = 9
+    num_categories: int = 12
     max_dep_level: int = 16
     max_ast_depth: int = 64
     max_sibling_index: int = 64

cppmega_mlx/data/domain_schema.py

@@ -81,6 +81,14 @@ class DomainEdgeKind(IntEnum):
     TYPE = 3
     DEF_USE = 4
     INCLUDE = 5
+    MACRO_PARAM_USE = 6
+    MACRO_INVOCATION = 7
+    MACRO_CONDITION = 8
+    MACRO_REDEFINITION = 9
+    MACRO_INCLUDE_ORDER = 10
+    MACRO_EXPANSION_CONDITION = 11
+    MACRO_EXPANSION_REDEFINITION = 12
+    MACRO_EXPANSION_INCLUDE_ORDER = 13
     BUILD_TARGET_DEP = 20
     BUILD_TARGET_SOURCE = 21
     BUILD_RULE_COMMAND = 22
@@ -118,10 +126,16 @@ class ParseConfidence(IntEnum):
     DomainKind.MAKE: ("MAKE_START", "MAKE_END"),
     DomainKind.NINJA: ("NINJA_START", "NINJA_END"),
     DomainKind.BAZEL: ("BAZEL_START", "BAZEL_END"),
-    # Autotools/Automake are Make-family text, but remain distinct in
-    # token_domain_ids so the model does not collapse them semantically.
-    DomainKind.AUTOCONF: ("MAKE_START", "MAKE_END"),
-    DomainKind.AUTOMAKE: ("MAKE_START", "MAKE_END"),
+    DomainKind.AUTOCONF: ("AUTOCONF_START", "AUTOCONF_END"),
+    DomainKind.AUTOMAKE: ("AUTOMAKE_START", "AUTOMAKE_END"),
+    DomainKind.MESON: ("MESON_START", "MESON_END"),
+    DomainKind.GN: ("GN_START", "GN_END"),
+    DomainKind.SCONS: ("SCONS_START", "SCONS_END"),
+    DomainKind.XMAKE: ("XMAKE_START", "XMAKE_END"),
+    DomainKind.COMPILE_COMMANDS: (
+        "COMPILE_COMMANDS_START",
+        "COMPILE_COMMANDS_END",
+    ),
     DomainKind.BASH: ("BASH_START", "BASH_END"),
     DomainKind.ZSH: ("ZSH_START", "ZSH_END"),
     DomainKind.SH: ("SH_START", "SH_END"),

cppmega_mlx/data/nanochat_pipeline/build_context.py

@@ -878,7 +878,7 @@ def read_text(name: str) -> str | None:
             path = compile_commands_path
         else:
             path = os.path.join(repo_dir, name)
-        if not os.path.exists(path):
+        if not os.path.isfile(path):
             return None
         with open(path, "r", encoding="utf-8", errors="replace") as f:
             return f.read()

cppmega_mlx/data/nanochat_pipeline/packed_rows_schema.py

@@ -81,6 +81,8 @@
 SOURCE_HAS_PR_DISCUSSIONS_COLUMN = "source_has_pr_discussions"
 SOURCE_PR_DISCUSSION_CHARS_COLUMN = "source_pr_discussion_chars"
 SOURCE_PR_DISCUSSION_LINES_COLUMN = "source_pr_discussion_lines"
+SOURCE_DOC_TYPES_COLUMN = "source_doc_types"
+SOURCE_HEADER_FRAGMENT_KINDS_COLUMN = "source_header_fragment_kinds"
 PACKED_REPO_COLUMN = REPO_COLUMN
 PACKED_FILEPATH_COLUMN = FILEPATH_COLUMN
 PACKED_COMMIT_HASH_COLUMN = COMMIT_HASH_COLUMN
@@ -212,6 +214,8 @@
     SOURCE_HAS_PR_DISCUSSIONS_COLUMN,
     SOURCE_PR_DISCUSSION_CHARS_COLUMN,
     SOURCE_PR_DISCUSSION_LINES_COLUMN,
+    SOURCE_DOC_TYPES_COLUMN,
+    SOURCE_HEADER_FRAGMENT_KINDS_COLUMN,
     ROW_PLATFORM_IDS_COLUMN,
     PACKED_REPO_COLUMN,
     PACKED_FILEPATH_COLUMN,
@@ -371,6 +375,8 @@ def resolve_packed_row_valid_token_count(
     "SOURCE_HAS_PR_DISCUSSIONS_COLUMN",
     "SOURCE_PR_DISCUSSION_CHARS_COLUMN",
     "SOURCE_PR_DISCUSSION_LINES_COLUMN",
+    "SOURCE_DOC_TYPES_COLUMN",
+    "SOURCE_HEADER_FRAGMENT_KINDS_COLUMN",
     "PLATFORM_IDS_COLUMN",
     "ROW_PLATFORM_IDS_COLUMN",
     "PACKED_REPO_COLUMN",

cppmega_mlx/data/nanochat_pipeline/tokenized_enriched.py

@@ -458,10 +458,12 @@ def _insert_domain_delimiters(
     domain: DomainKind,
     insert_at: int = 1,
 ) -> None:
+    if DomainKind(domain) == DomainKind.UNKNOWN:
+        return
     try:
         start_id, end_id = delimiter_token_ids(domain)
-    except KeyError:
-        return
+    except KeyError as exc:
+        raise ValueError(f"missing delimiter token contract for domain {domain!r}") from exc
     token_ids = list(row[TOKEN_IDS_COLUMN])
     if insert_at > len(token_ids):
         insert_at = len(token_ids)

cppmega_mlx/data/nanochat_pipeline/tokenized_enriched_schema.py

@@ -96,6 +96,9 @@ def _extract_span_bounds(span: object) -> tuple[int, int] | None:
 HUNK_ID_PER_TOKEN_COLUMN = "hunk_id_per_token"
 EDIT_OP_PER_TOKEN_COLUMN = "edit_op_per_token"
 
+DOC_TYPE_COLUMN = "doc_type"
+HEADER_FRAGMENT_KIND_COLUMN = "header_fragment_kind"
+
 REPO_COLUMN = "repo"
 FILEPATH_COLUMN = "filepath"
 COMMIT_HASH_COLUMN = "commit_hash"
@@ -351,6 +354,8 @@ def has_materialized_token_metadata(
     "CHANGED_CHUNK_SPANS_COLUMN",
     "HUNK_ID_PER_TOKEN_COLUMN",
     "EDIT_OP_PER_TOKEN_COLUMN",
+    "DOC_TYPE_COLUMN",
+    "HEADER_FRAGMENT_KIND_COLUMN",
     "REPO_COLUMN",
     "FILEPATH_COLUMN",
     "COMMIT_HASH_COLUMN",

cppmega_mlx/data/tokenizer_contract.py

@@ -38,6 +38,20 @@
     "NINJA_END": 198,
     "BAZEL_START": 199,
     "BAZEL_END": 200,
+    "AUTOCONF_START": 223,
+    "AUTOCONF_END": 224,
+    "AUTOMAKE_START": 225,
+    "AUTOMAKE_END": 226,
+    "MESON_START": 227,
+    "MESON_END": 228,
+    "GN_START": 229,
+    "GN_END": 230,
+    "SCONS_START": 231,
+    "SCONS_END": 232,
+    "XMAKE_START": 233,
+    "XMAKE_END": 234,
+    "COMPILE_COMMANDS_START": 235,
+    "COMPILE_COMMANDS_END": 236,
     "BASH_START": 201,
     "BASH_END": 202,
     "ZSH_START": 203,

cppmega_mlx/models/dense_cpp_lm.py

@@ -88,7 +88,7 @@ class DenseCppLMConfig:
     # Side-channel structure embedding (CppMegaStructureEmbedding).
     structure_components: str = "all"
     structure_bottleneck_dim: int = 128
-    structure_num_categories: int = 9
+    structure_num_categories: int = 12
     structure_max_dep_level: int = 64
     structure_max_ast_depth: int = 128
     structure_max_sibling_index: int = 128

cppmega_mlx/models/hybrid_lm.py

@@ -240,7 +240,7 @@ class HybridTinyConfig:
     structure_vocab_size: int = 32
     structure_components: str = "core"
     structure_bottleneck_dim: int = 64
-    structure_num_categories: int = 9
+    structure_num_categories: int = 12
     structure_max_dep_level: int = 16
     structure_max_ast_depth: int = 64
     structure_max_sibling_index: int = 64

cppmega_mlx/nn/domain_graph_routes.py

@@ -82,6 +82,14 @@ def build_domain_block_candidates(
 DEFAULT_EDGE_WEIGHTS: dict[int, float] = {
     int(DomainEdgeKind.BUILD_TARGET_SOURCE): 1.0,
     int(DomainEdgeKind.BUILD_TARGET_DEP): 1.0,
+    int(DomainEdgeKind.MACRO_PARAM_USE): 1.0,
+    int(DomainEdgeKind.MACRO_INVOCATION): 1.0,
+    int(DomainEdgeKind.MACRO_CONDITION): 1.0,
+    int(DomainEdgeKind.MACRO_REDEFINITION): 1.0,
+    int(DomainEdgeKind.MACRO_INCLUDE_ORDER): 1.0,
+    int(DomainEdgeKind.MACRO_EXPANSION_CONDITION): 1.0,
+    int(DomainEdgeKind.MACRO_EXPANSION_REDEFINITION): 1.0,
+    int(DomainEdgeKind.MACRO_EXPANSION_INCLUDE_ORDER): 1.0,
     int(DomainEdgeKind.SHELL_PIPE): 1.0,
     int(DomainEdgeKind.DIAG_PRIMARY_LOCATION): 2.0,
     int(DomainEdgeKind.DIAG_FIXIT): 2.0,