Add domain routed data pipeline and graph sidecars

Author: apstenku123

.gitattributes

@@ -0,0 +1,2 @@
+outputs/**/*.bin filter=lfs diff=lfs merge=lfs -text
+outputs/conveyor/**/*.jsonl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -118,6 +118,7 @@ outputs/sidecar_audit_commit_pr_8k_verified/sidecar_parquet_audit.md
 outputs/sidecar_audit_pr_1k_2k_4k_8k/sidecar_parquet_audit.json
 outputs/sidecar_audit_pr_1k_2k_4k_8k/sidecar_parquet_audit.md
 outputs/conveyor/progress_code.jsonl
+outputs/conveyor/extract_cache/kodi/kodi_commits.jsonl
 outputs/conveyor/progress_commits.jsonl
 outputs/conveyor/locks/code.lock
 outputs/conveyor/locks/commits.lock

configs/stage_domain_routed_foundation.yaml

@@ -0,0 +1,102 @@
+# Domain-routed cppmega world-code training recipe.
+#
+# This config is intentionally declarative: runners may map these weights into
+# their own task-mixer implementation, but graph routes are not optional for
+# cppmega world/code model runs.
+
+tokenizer_contract: cppmega_mlx/tokenizer/tokenizer_contract_v1.json
+sequence_curriculum: [1024, 2048, 4096, 8192, 16384]
+
+requirements:
+  require_domain_delimiters: true
+  require_token_sidecars:
+    - token_domain_ids
+    - token_role_ids
+    - token_confidence_ids
+    - token_structure_ids
+    - token_dep_levels
+  require_graph_sidecars:
+    - token_call_edges
+    - token_type_edges
+    - token_domain_edges
+    - token_build_edges
+    - token_shell_edges
+    - token_diagnostic_edges
+    - token_cross_domain_edges
+  allow_token_only_baseline: false
+
+model_inputs:
+  side_embeddings:
+    domain_ids: true
+    role_ids: true
+    confidence_ids: true
+    structure_ids: true
+    dep_levels: true
+    ast_depth: true
+  graph_bias:
+    dense_gqa: true
+    dsa_topk: true
+    beta_code: 1.0
+    beta_build: 1.0
+    beta_shell: 0.75
+    beta_diagnostic: 2.0
+    beta_cross_domain: 2.0
+
+stages:
+  stage1_foundation:
+    mix:
+      cpp_code: 0.58
+      build_files: 0.10
+      shell_scripts: 0.04
+      diagnostics: 0.03
+      fim_ifim: 0.15
+      graph_indexer: 0.10
+    objectives:
+      causal_lm: 1.0
+      opener_domain_classification: 0.05
+      graph_edge_prediction: 0.05
+      graph_indexer_bce: 0.10
+      graph_indexer_coverage: 0.05
+  stage2_repair_midtrain:
+    mix:
+      cpp_code: 0.20
+      commit_pr_transform: 0.45
+      build_diagnostics: 0.15
+      fim_ifim: 0.15
+      graph_indexer: 0.05
+    objectives:
+      causal_lm: 1.0
+      diff_prediction: 0.25
+      cross_domain_retrieval: 0.10
+      diagnostic_conditioned_ifim: 0.20
+  stage3_world_trajectory:
+    mix:
+      tool_trajectories: 0.35
+      compile_test_observations: 0.25
+      commit_pr_transform: 0.25
+      cpp_code: 0.15
+    objectives:
+      transition_model: 0.50
+      reward_prediction: 0.05
+      next_action_prediction: 0.15
+      patch_generation: 0.30
+
+domains:
+  build_kinds:
+    - cmake
+    - make
+    - ninja
+    - bazel
+    - autoconf
+    - automake
+    - meson
+    - gn
+    - scons
+    - xmake
+  shell_kinds:
+    - sh
+    - bash
+    - zsh
+    - tcsh
+  diagnostics:
+    raw_without_edges_allowed_only_with_parse_confidence_raw: true

cppmega_mlx/data/agent_trajectory.py

@@ -37,6 +37,14 @@
 from pathlib import Path
 from typing import Any
 
+from cppmega_mlx.data.build_parsers.base import ParsedDomainDocument
+from cppmega_mlx.data.diagnostic_parsers import (
+    parse_build_error,
+    parse_clang_diagnostic,
+    parse_linker_error,
+)
+from cppmega_mlx.data.shell_parsers import parse_bash, parse_sh, parse_tcsh, parse_zsh
+
 # --------------------------------------------------------------------------- #
 # Action-kind classification
 # --------------------------------------------------------------------------- #
@@ -116,6 +124,59 @@ def classify_action(tool_name: str, command: str | None) -> str:
     return ACTION_OTHER
 
 
+def parse_shell_action_domain(
+    command: str,
+    *,
+    shell_kind: str | None = None,
+) -> ParsedDomainDocument:
+    """Parse a trajectory shell action into the matching shell domain.
+
+    Unknown shell stays POSIX ``sh``. We do not label it bash unless the caller
+    or command itself gives evidence, because bash/zsh/tcsh have different
+    syntax and should remain separate domains for training.
+    """
+
+    kind = (shell_kind or "").strip().lower()
+    stripped = command.lstrip()
+    if not kind:
+        if stripped.startswith("#!"):
+            first = stripped.splitlines()[0]
+            if "zsh" in first:
+                kind = "zsh"
+            elif "tcsh" in first or "csh" in first:
+                kind = "tcsh"
+            elif "bash" in first:
+                kind = "bash"
+            else:
+                kind = "sh"
+        else:
+            head = stripped.split(maxsplit=1)[0] if stripped else ""
+            kind = head if head in {"bash", "zsh", "tcsh", "sh"} else "sh"
+
+    if kind == "bash":
+        return parse_bash(command)
+    if kind == "zsh":
+        return parse_zsh(command)
+    if kind == "tcsh":
+        return parse_tcsh(command)
+    return parse_sh(command)
+
+
+def parse_result_diagnostic_domain(result_text: str) -> ParsedDomainDocument | None:
+    """Parse a build/test/tool result into a diagnostic domain when possible."""
+
+    if not result_text.strip():
+        return None
+    lower = result_text.lower()
+    if "undefined reference" in lower or "unresolved external symbol" in lower:
+        return parse_linker_error(result_text)
+    if re.search(r"^[^\n:]+:\d+:\d+:\s+(fatal error|error|warning|note):", result_text, re.MULTILINE):
+        return parse_clang_diagnostic(result_text)
+    if "cmake error" in lower or "ninja:" in lower or "build stopped" in lower:
+        return parse_build_error(result_text)
+    return None
+
+
 # --------------------------------------------------------------------------- #
 # Transition record (flat parquet-friendly schema)
 # --------------------------------------------------------------------------- #
@@ -799,6 +860,8 @@ def write_parquet(transitions: list[AgentTransition], out_path: Path) -> Path:
     "enumerate_remote_sessions",
     "extract_all",
     "extract_session",
+    "parse_result_diagnostic_domain",
+    "parse_shell_action_domain",
     "walk_claude",
     "walk_codex",
     "write_parquet",

cppmega_mlx/data/build_parsers/__init__.py

@@ -0,0 +1,16 @@
+"""Deterministic build-system domain parsers."""
+
+from cppmega_mlx.data.build_parsers.autotools import parse_autoconf, parse_automake
+from cppmega_mlx.data.build_parsers.bazel import parse_bazel
+from cppmega_mlx.data.build_parsers.cmake import parse_cmake
+from cppmega_mlx.data.build_parsers.make import parse_make
+from cppmega_mlx.data.build_parsers.ninja import parse_ninja
+
+__all__ = [
+    "parse_autoconf",
+    "parse_automake",
+    "parse_bazel",
+    "parse_cmake",
+    "parse_make",
+    "parse_ninja",
+]

cppmega_mlx/data/build_parsers/autotools.py

@@ -0,0 +1,42 @@
+"""Autoconf / Automake build-domain parser."""
+
+from __future__ import annotations
+
+from cppmega_mlx.data.build_parsers.base import ParsedDomainDocument
+from cppmega_mlx.data.build_parsers.make import parse_automake
+from cppmega_mlx.data.domain_schema import (
+    DomainKind,
+    DomainRoleKind,
+    ParseConfidence,
+)
+
+
+_AUTOCONF_MACROS = {
+    "AC_INIT",
+    "AC_CONFIG_FILES",
+    "AC_PROG_CC",
+    "AC_PROG_CXX",
+    "AM_INIT_AUTOMAKE",
+    "PKG_CHECK_MODULES",
+}
+
+
+def parse_autoconf(text: str) -> ParsedDomainDocument:
+    doc = ParsedDomainDocument.new(
+        domain=DomainKind.AUTOCONF,
+        text=text,
+        confidence=ParseConfidence.HEURISTIC,
+        metadata={"build_kind": "autoconf"},
+    )
+    next_entity = 1
+    for idx, token in enumerate(doc.tokens):
+        value = token.text
+        if value in _AUTOCONF_MACROS or value.startswith("AC_") or value.startswith("AM_"):
+            doc.set_role(idx, DomainRoleKind.COMMAND, entity=next_entity)
+            next_entity += 1
+        elif value.startswith("$"):
+            doc.set_role(idx, DomainRoleKind.VARIABLE)
+    return doc
+
+
+__all__ = ["parse_autoconf", "parse_automake"]