Skip to content

Commit d88e722

Browse files
apstenku123apstenku123
authored
Merge baseline corpus and MLX review hardening
Self-hosted macOS/Linux contracts and self-hosted CodeQL are green. Includes clean-checkout portability fixes found by the runners and preserves individual commits for the follow-up case branches.
2 parents dd2b703 + 8a8de80 commit d88e722

34 files changed

Lines changed: 2068 additions & 376 deletions

‎.github/workflows/ci-self-hosted.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: CppMega MLX self-hosted CI
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
push:
7+
branches: [main]
8+
workflow_dispatch:
9+
10+
concurrency:
11+
group: cppmega-mlx-ci-${{ github.ref }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
mac-contracts:
16+
name: macOS MLX data, model, and eval contracts
17+
runs-on: [self-hosted, macOS, ARM64, cppmega-mlx-macos]
18+
timeout-minutes: 30
19+
steps:
20+
- uses: actions/checkout@v4
21+
- name: Run focused MLX contract suite
22+
shell: bash
23+
run: |
24+
set -euo pipefail
25+
python_bin=/Volumes/external/sources/cppmega.mlx/.venv/bin/python
26+
test -x "${python_bin}"
27+
"${python_bin}" -m pytest -q \
28+
tests/test_ast_fim.py \
29+
tests/test_audit_sidecar_parquet.py \
30+
tests/test_cpp_jsonl_generation_compile_eval.py \
31+
tests/test_data_package_imports.py \
32+
tests/test_domain_graph_routes.py \
33+
tests/test_eval_domain_routed_codegen.py \
34+
tests/test_inference_generation.py \
35+
tests/test_megatron_indexed.py \
36+
tests/test_pack_enriched_rows.py \
37+
tests/test_streaming_conveyor_progress.py \
38+
tests/test_process_commits_fail_loud.py \
39+
tests/test_repair_packed_document_boundaries.py
40+
git diff --check
41+
42+
linux-portable:
43+
name: Linux portable syntax and conveyor contracts
44+
runs-on: [self-hosted, Linux, X64, cppmega-mlx]
45+
timeout-minutes: 20
46+
steps:
47+
- uses: actions/checkout@v4
48+
- uses: actions/setup-python@v5
49+
with:
50+
python-version: "3.13"
51+
- name: Install portable test dependencies
52+
run: python -m pip install --disable-pip-version-check pytest numpy pyarrow tokenizers zstandard
53+
- name: Run portable script contracts
54+
shell: bash
55+
run: |
56+
set -euo pipefail
57+
# cppmega.mlx imports Apple's MLX runtime at package import time. The
58+
# Linux runner exercises only portable scripts and deliberately skips
59+
# the macOS-only root conftest/package initialization.
60+
python -m pytest -q --noconftest \
61+
tests/test_audit_sidecar_parquet.py \
62+
tests/test_data_package_imports.py \
63+
tests/test_streaming_conveyor_progress.py \
64+
tests/test_process_commits_fail_loud.py \
65+
tests/test_repair_packed_document_boundaries.py
66+
python -m compileall -q scripts tools/clang_indexer
67+
git diff --check

‎.github/workflows/cuda-lane.yml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
name: CUDA Lane
22

33
on:
4-
push:
5-
branches: [main]
6-
pull_request:
7-
branches: [main]
4+
# CUDA capacity is provisioned explicitly (for example on Nebius H200).
5+
# Do not leave ordinary PRs queued forever when no self-hosted CUDA runner
6+
# is attached to this repository.
87
workflow_dispatch:
98

109
jobs:

‎AGENTS.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,12 @@ See CLAUDE.md. This overrides every rule below.
2828
format directly;
2929
- only use casts/reshapes as explicit graph decisions, not hidden adapter
3030
staging.
31+
32+
## Native subagents
33+
34+
- Use Codex native subagents for independent, bounded parallel work when it
35+
materially improves throughput.
36+
- Every native subagent must use `gpt-5.6-sol` with `ultra` or `max`
37+
reasoning. Use `ultra` by default.
38+
- Do not use another model or a lower reasoning level unless the user
39+
explicitly overrides this rule.

‎cppmega_mlx/data/__init__.py‎

Lines changed: 88 additions & 135 deletions
Original file line numberDiff line numberDiff line change
@@ -1,137 +1,90 @@
1-
"""Data readers and batch collation helpers."""
1+
"""Data readers and batch collation helpers.
22
3-
from cppmega_mlx.data.batch import LMTokenBatch, ensure_lm_batch, synthetic_token_batch
4-
from cppmega_mlx.data.dataloader_bridge import (
5-
LocalTokenBatchDataset,
6-
TorchDataLoaderBridgeConfig,
7-
TorchDataLoaderBridgeError,
8-
build_spawn_dataloader,
9-
is_torch_dataloader_available,
10-
iter_mlx_batches,
11-
)
12-
from cppmega_mlx.data.fim import (
13-
EOT_ID,
14-
FIMSpecialTokenIds,
15-
FIM_INSTRUCTION_ID,
16-
FIMMode,
17-
FIM_MIDDLE_ID,
18-
FIM_PREFIX_ID,
19-
FIM_SPECIAL_TOKEN_IDS,
20-
FIM_SUFFIX_ID,
21-
apply_fim_permutation,
22-
apply_fim_transform,
23-
apply_ifim_permutation,
24-
apply_ifim_transform,
25-
extract_ifim_instruction_text,
26-
sample_middle_span,
27-
)
28-
from cppmega_mlx.data.megatron_indexed import (
29-
MegatronIndexedDataset,
30-
MegatronIndexedMetadata,
31-
MegatronIndexedMultiShardDataset,
32-
MegatronIndexedMultiShardMetadata,
33-
megatron_indexed_side_channel_schema,
34-
open_megatron_indexed_dataset,
35-
)
36-
from cppmega_mlx.data.packing import (
37-
OversizedSamplePolicy,
38-
PackedSequences,
39-
PackingStrategy,
40-
cumulative_doc_ids_from_eos,
41-
document_boundary_mask,
42-
mlx_cumulative_doc_ids_from_eos,
43-
mlx_document_boundary_mask,
44-
mlx_sequence_packing_attention_mask,
45-
pack_bos_aligned_best_fit,
46-
pack_documents_with_eos,
47-
)
48-
from cppmega_mlx.data.parquet_dataset import (
49-
MultiShardTokenParquetDataset,
50-
ParquetColumns,
51-
TokenParquetDataset,
52-
)
53-
from cppmega_mlx.data.platform_context import (
54-
MAX_PLATFORM_IDS,
55-
PLATFORM_VOCAB,
56-
PLATFORM_VOCAB_SIZE,
57-
PlatformContext,
58-
encode_platform_context,
59-
parse_platform_context,
60-
platform_ids_array,
61-
render_platform_context,
62-
)
63-
from cppmega_mlx.data.tokenizer_contract import (
64-
REQUIRED_SPECIAL_TOKEN_IDS,
65-
SpecialTokenMapping,
66-
TOOL_USE_SPECIAL_TOKEN_IDS,
67-
validate_required_special_token_ids,
68-
)
69-
from cppmega_mlx.data.token_dataset import (
70-
BatchCursor,
71-
TokenDatasetMetadata,
72-
TokenNpzDataset,
73-
iterate_token_batches,
74-
open_token_dataset,
75-
)
3+
The package also contains portable corpus/indexer modules used on Linux hosts
4+
where Apple's MLX runtime is unavailable. Public MLX-backed exports are loaded
5+
on first attribute access so importing a portable submodule does not eagerly
6+
import the entire training runtime.
7+
"""
768

77-
__all__ = [
78-
"BatchCursor",
79-
"EOT_ID",
80-
"FIMSpecialTokenIds",
81-
"FIM_INSTRUCTION_ID",
82-
"FIMMode",
83-
"FIM_MIDDLE_ID",
84-
"FIM_PREFIX_ID",
85-
"FIM_SPECIAL_TOKEN_IDS",
86-
"FIM_SUFFIX_ID",
87-
"LMTokenBatch",
88-
"LocalTokenBatchDataset",
89-
"MAX_PLATFORM_IDS",
90-
"MegatronIndexedDataset",
91-
"MegatronIndexedMetadata",
92-
"MegatronIndexedMultiShardDataset",
93-
"MegatronIndexedMultiShardMetadata",
94-
"MultiShardTokenParquetDataset",
95-
"OversizedSamplePolicy",
96-
"PLATFORM_VOCAB",
97-
"PLATFORM_VOCAB_SIZE",
98-
"ParquetColumns",
99-
"PackedSequences",
100-
"PackingStrategy",
101-
"PlatformContext",
102-
"REQUIRED_SPECIAL_TOKEN_IDS",
103-
"SpecialTokenMapping",
104-
"TokenDatasetMetadata",
105-
"TorchDataLoaderBridgeConfig",
106-
"TorchDataLoaderBridgeError",
107-
"TOOL_USE_SPECIAL_TOKEN_IDS",
108-
"TokenNpzDataset",
109-
"TokenParquetDataset",
110-
"apply_fim_permutation",
111-
"apply_fim_transform",
112-
"apply_ifim_permutation",
113-
"apply_ifim_transform",
114-
"build_spawn_dataloader",
115-
"cumulative_doc_ids_from_eos",
116-
"document_boundary_mask",
117-
"encode_platform_context",
118-
"ensure_lm_batch",
119-
"extract_ifim_instruction_text",
120-
"is_torch_dataloader_available",
121-
"iter_mlx_batches",
122-
"iterate_token_batches",
123-
"megatron_indexed_side_channel_schema",
124-
"mlx_cumulative_doc_ids_from_eos",
125-
"mlx_document_boundary_mask",
126-
"mlx_sequence_packing_attention_mask",
127-
"open_megatron_indexed_dataset",
128-
"open_token_dataset",
129-
"pack_bos_aligned_best_fit",
130-
"pack_documents_with_eos",
131-
"parse_platform_context",
132-
"platform_ids_array",
133-
"render_platform_context",
134-
"sample_middle_span",
135-
"synthetic_token_batch",
136-
"validate_required_special_token_ids",
137-
]
9+
from __future__ import annotations
10+
11+
from importlib import import_module
12+
from typing import Any
13+
14+
15+
_EXPORT_MODULES = {
16+
"LMTokenBatch": "cppmega_mlx.data.batch",
17+
"ensure_lm_batch": "cppmega_mlx.data.batch",
18+
"synthetic_token_batch": "cppmega_mlx.data.batch",
19+
"LocalTokenBatchDataset": "cppmega_mlx.data.dataloader_bridge",
20+
"TorchDataLoaderBridgeConfig": "cppmega_mlx.data.dataloader_bridge",
21+
"TorchDataLoaderBridgeError": "cppmega_mlx.data.dataloader_bridge",
22+
"build_spawn_dataloader": "cppmega_mlx.data.dataloader_bridge",
23+
"is_torch_dataloader_available": "cppmega_mlx.data.dataloader_bridge",
24+
"iter_mlx_batches": "cppmega_mlx.data.dataloader_bridge",
25+
"EOT_ID": "cppmega_mlx.data.fim",
26+
"FIMSpecialTokenIds": "cppmega_mlx.data.fim",
27+
"FIM_INSTRUCTION_ID": "cppmega_mlx.data.fim",
28+
"FIMMode": "cppmega_mlx.data.fim",
29+
"FIM_MIDDLE_ID": "cppmega_mlx.data.fim",
30+
"FIM_PREFIX_ID": "cppmega_mlx.data.fim",
31+
"FIM_SPECIAL_TOKEN_IDS": "cppmega_mlx.data.fim",
32+
"FIM_SUFFIX_ID": "cppmega_mlx.data.fim",
33+
"apply_fim_permutation": "cppmega_mlx.data.fim",
34+
"apply_fim_transform": "cppmega_mlx.data.fim",
35+
"apply_ifim_permutation": "cppmega_mlx.data.fim",
36+
"apply_ifim_transform": "cppmega_mlx.data.fim",
37+
"extract_ifim_instruction_text": "cppmega_mlx.data.fim",
38+
"sample_middle_span": "cppmega_mlx.data.fim",
39+
"MegatronIndexedDataset": "cppmega_mlx.data.megatron_indexed",
40+
"MegatronIndexedMetadata": "cppmega_mlx.data.megatron_indexed",
41+
"MegatronIndexedMultiShardDataset": "cppmega_mlx.data.megatron_indexed",
42+
"MegatronIndexedMultiShardMetadata": "cppmega_mlx.data.megatron_indexed",
43+
"megatron_indexed_side_channel_schema": "cppmega_mlx.data.megatron_indexed",
44+
"open_megatron_indexed_dataset": "cppmega_mlx.data.megatron_indexed",
45+
"OversizedSamplePolicy": "cppmega_mlx.data.packing",
46+
"PackedSequences": "cppmega_mlx.data.packing",
47+
"PackingStrategy": "cppmega_mlx.data.packing",
48+
"cumulative_doc_ids_from_eos": "cppmega_mlx.data.packing",
49+
"document_boundary_mask": "cppmega_mlx.data.packing",
50+
"mlx_cumulative_doc_ids_from_eos": "cppmega_mlx.data.packing",
51+
"mlx_document_boundary_mask": "cppmega_mlx.data.packing",
52+
"mlx_sequence_packing_attention_mask": "cppmega_mlx.data.packing",
53+
"pack_bos_aligned_best_fit": "cppmega_mlx.data.packing",
54+
"pack_documents_with_eos": "cppmega_mlx.data.packing",
55+
"MultiShardTokenParquetDataset": "cppmega_mlx.data.parquet_dataset",
56+
"ParquetColumns": "cppmega_mlx.data.parquet_dataset",
57+
"TokenParquetDataset": "cppmega_mlx.data.parquet_dataset",
58+
"MAX_PLATFORM_IDS": "cppmega_mlx.data.platform_context",
59+
"PLATFORM_VOCAB": "cppmega_mlx.data.platform_context",
60+
"PLATFORM_VOCAB_SIZE": "cppmega_mlx.data.platform_context",
61+
"PlatformContext": "cppmega_mlx.data.platform_context",
62+
"encode_platform_context": "cppmega_mlx.data.platform_context",
63+
"parse_platform_context": "cppmega_mlx.data.platform_context",
64+
"platform_ids_array": "cppmega_mlx.data.platform_context",
65+
"render_platform_context": "cppmega_mlx.data.platform_context",
66+
"REQUIRED_SPECIAL_TOKEN_IDS": "cppmega_mlx.data.tokenizer_contract",
67+
"SpecialTokenMapping": "cppmega_mlx.data.tokenizer_contract",
68+
"TOOL_USE_SPECIAL_TOKEN_IDS": "cppmega_mlx.data.tokenizer_contract",
69+
"validate_required_special_token_ids": "cppmega_mlx.data.tokenizer_contract",
70+
"BatchCursor": "cppmega_mlx.data.token_dataset",
71+
"TokenDatasetMetadata": "cppmega_mlx.data.token_dataset",
72+
"TokenNpzDataset": "cppmega_mlx.data.token_dataset",
73+
"iterate_token_batches": "cppmega_mlx.data.token_dataset",
74+
"open_token_dataset": "cppmega_mlx.data.token_dataset",
75+
}
76+
77+
__all__ = list(_EXPORT_MODULES)
78+
79+
80+
def __getattr__(name: str) -> Any:
81+
module_name = _EXPORT_MODULES.get(name)
82+
if module_name is None:
83+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
84+
value = getattr(import_module(module_name), name)
85+
globals()[name] = value
86+
return value
87+
88+
89+
def __dir__() -> list[str]:
90+
return sorted(set(globals()) | set(__all__))

‎cppmega_mlx/data/ast_fim.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@
2222
* 10% of the time we emit a random-CHAR-FIM example whose middle is a random
2323
token span (``fim.sample_middle_span``) that may start/end mid-statement —
2424
this teaches robustness to arbitrary cursor positions. The fall-back to the
25-
char path is RECORDED in the returned ``AstFimResult.kind`` ("char_fim"),
26-
never silent.
25+
char path is RECORDED in the returned ``AstFimResult.kind`` ("char_fim"
26+
or "char_ifim" for instruction-aware examples), never silent.
2727
2828
iFIM variant: when the document carries a leading comment / docstring we extract
2929
it via ``fim.extract_ifim_instruction_text`` and emit through
@@ -58,7 +58,7 @@
5858
# Fraction of AST-FIM examples; the remaining slice is random-char-FIM.
5959
DEFAULT_AST_FIM_RATE = 0.9
6060

61-
AstFimKind = Literal["ast_fim", "char_fim", "ast_ifim"]
61+
AstFimKind = Literal["ast_fim", "char_fim", "ast_ifim", "char_ifim"]
6262

6363

6464
class NoEligibleChunkError(ValueError):
@@ -306,11 +306,11 @@ def apply_ast_ifim(
306306
except NoEligibleChunkError:
307307
start, end = sample_middle_span(length, rng=rand)
308308
chunk_index = None
309-
kind = "char_fim"
309+
kind = "char_ifim"
310310
else:
311311
start, end = sample_middle_span(length, rng=rand)
312312
chunk_index = None
313-
kind = "char_fim"
313+
kind = "char_ifim"
314314

315315
mode: FIMMode = "spm" if rand.random() < spm_rate else "psm"
316316
permuted = apply_ifim_permutation(

0 commit comments

Comments
 (0)