Harden corpus packing and graph bundle ingress

.github/workflows/ci-self-hosted.yml

@@ -0,0 +1,62 @@
+name: CppMega MLX self-hosted CI
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: cppmega-mlx-ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  mac-contracts:
+    name: macOS MLX data, model, and eval contracts
+    runs-on: [self-hosted, macOS, ARM64, cppmega-mlx-macos]
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run focused MLX contract suite
+        shell: bash
+        run: |
+          set -euo pipefail
+          python_bin=/Volumes/external/sources/cppmega.mlx/.venv/bin/python
+          test -x "${python_bin}"
+          "${python_bin}" -m pytest -q \
+            tests/test_ast_fim.py \
+            tests/test_audit_sidecar_parquet.py \
+            tests/test_cpp_jsonl_generation_compile_eval.py \
+            tests/test_domain_graph_routes.py \
+            tests/test_eval_domain_routed_codegen.py \
+            tests/test_inference_generation.py \
+            tests/test_megatron_indexed.py \
+            tests/test_pack_enriched_rows.py \
+            tests/test_streaming_conveyor_progress.py \
+            tests/test_process_commits_fail_loud.py \
+            tests/test_repair_packed_document_boundaries.py
+          git diff --check
+
+  linux-portable:
+    name: Linux portable conveyor contracts
+    runs-on: [self-hosted, Linux, X64, cppmega-mlx]
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - name: Install portable test dependencies
+        run: python -m pip install --disable-pip-version-check pytest numpy pyarrow tokenizers zstandard
+      - name: Run portable script contracts
+        shell: bash
+        run: |
+          set -euo pipefail
+          python -m pytest -q \
+            tests/test_audit_sidecar_parquet.py \
+            tests/test_pack_enriched_rows.py \
+            tests/test_streaming_conveyor_progress.py \
+            tests/test_process_commits_fail_loud.py \
+            tests/test_repair_packed_document_boundaries.py
+          git diff --check

cppmega_mlx/data/ast_fim.py

@@ -22,8 +22,8 @@
   * 10% of the time we emit a random-CHAR-FIM example whose middle is a random
     token span (``fim.sample_middle_span``) that may start/end mid-statement —
     this teaches robustness to arbitrary cursor positions.  The fall-back to the
-    char path is RECORDED in the returned ``AstFimResult.kind`` ("char_fim"),
-    never silent.
+    char path is RECORDED in the returned ``AstFimResult.kind`` ("char_fim"
+    or "char_ifim" for instruction-aware examples), never silent.
 
 iFIM variant: when the document carries a leading comment / docstring we extract
 it via ``fim.extract_ifim_instruction_text`` and emit through
@@ -58,7 +58,7 @@
 # Fraction of AST-FIM examples; the remaining slice is random-char-FIM.
 DEFAULT_AST_FIM_RATE = 0.9
 
-AstFimKind = Literal["ast_fim", "char_fim", "ast_ifim"]
+AstFimKind = Literal["ast_fim", "char_fim", "ast_ifim", "char_ifim"]
 
 
 class NoEligibleChunkError(ValueError):
@@ -306,11 +306,11 @@ def apply_ast_ifim(
         except NoEligibleChunkError:
             start, end = sample_middle_span(length, rng=rand)
             chunk_index = None
-            kind = "char_fim"
+            kind = "char_ifim"
     else:
         start, end = sample_middle_span(length, rng=rand)
         chunk_index = None
-        kind = "char_fim"
+        kind = "char_ifim"
 
     mode: FIMMode = "spm" if rand.random() < spm_rate else "psm"
     permuted = apply_ifim_permutation(