Skip to content

Commit f539af5

Browse files
committed
feat(v4): 5 new bricks from open mlx-lm PRs — Mistral4 MLA, DSv4, Bailing/Ling
Direct re-export of new model bricks from open mlx-lm PRs, vendored into our editable mlx-lm checkout at /Users/dave/sources/mlx-lm on branch cppmega-integration (pushed to DatasunriseOU/mlx-lm). Each is wrapped as a thin nn.Module that exposes ``self.inner`` so weight loaders find upstream parameter paths at the expected locations. PRs integrated (commits 96febcc..5f4e5e5 in our mlx-lm fork): - #1037 Mistral Small 4 — Absorbed MLA + INT4 latent cache -> "mistral4_mla" block kind, Mistral4MLABlock - #1057 LongCat Next — full model class, loadable directly via mlx_lm.load (no per-brick wrapper needed) - #1227 Bailing/Ling-2.6 — LinearAttention + MultiLatentAttention + MoE -> "bailing_linear" / "bailing_mla" / "bailing_moe" block kinds - #1201 DeepSeek-V4 Flash — hash-indexed sparse attention + block -> "dsv4_attention" block kind All 5 bricks registered in BLOCK_BUILDERS for UnifiedSuperblock composition (this is the modular GUI-driven model construction surface). cppmega_v4/nn/mlx_lm_bricks.py: 5 wrapper classes + minimal-stub ModelArgs builders so each upstream class can be instantiated standalone. cppmega_v4/models/unified_superblock_v4.py: 5 _build_<kind> functions added; BLOCK_BUILDERS dict extended. tests/v4/test_mlx_lm_bricks.py: 14 tests covering direct-import-from-mlx-lm, forward shape, BLOCK_BUILDERS round-trip for each brick. Suite: 393 passed / 0 skipped (was 379).
1 parent f6798a1 commit f539af5

3 files changed

Lines changed: 542 additions & 0 deletions

File tree

cppmega_v4/models/unified_superblock_v4.py

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,56 @@ def _build_mla(hidden_size: int, params: dict) -> nn.Module:
137137
return MLABlock(cfg)
138138

139139

140+
def _build_mistral4_mla(hidden_size: int, params: dict) -> nn.Module:
141+
"""Mistral Small 4 absorbed MLA + INT4 latent cache (mlx-lm PR #1037)."""
142+
from cppmega_v4.nn.mlx_lm_bricks import Mistral4MLABlock, Mistral4MLAConfig
143+
cfg = Mistral4MLAConfig(hidden_size=hidden_size, **{
144+
k: v for k, v in params.items()
145+
if k in Mistral4MLAConfig.__dataclass_fields__
146+
})
147+
return Mistral4MLABlock(cfg)
148+
149+
150+
def _build_dsv4_attention(hidden_size: int, params: dict) -> nn.Module:
151+
"""DeepSeek-V4 (Flash) hash-indexed sparse attention (mlx-lm PR #1201)."""
152+
from cppmega_v4.nn.mlx_lm_bricks import DSv4AttentionBlock, DSv4AttentionConfig
153+
cfg = DSv4AttentionConfig(hidden_size=hidden_size, **{
154+
k: v for k, v in params.items()
155+
if k in DSv4AttentionConfig.__dataclass_fields__
156+
})
157+
return DSv4AttentionBlock(cfg)
158+
159+
160+
def _build_bailing_linear(hidden_size: int, params: dict) -> nn.Module:
161+
"""Ling-2.6-flash linear attention (mlx-lm PR #1227)."""
162+
from cppmega_v4.nn.mlx_lm_bricks import BailingLinearAttnBlock, BailingLinearConfig
163+
cfg = BailingLinearConfig(hidden_size=hidden_size, **{
164+
k: v for k, v in params.items()
165+
if k in BailingLinearConfig.__dataclass_fields__
166+
})
167+
return BailingLinearAttnBlock(cfg)
168+
169+
170+
def _build_bailing_mla(hidden_size: int, params: dict) -> nn.Module:
171+
"""Ling-2.6-flash multi-latent attention (mlx-lm PR #1227)."""
172+
from cppmega_v4.nn.mlx_lm_bricks import BailingMLABlock, BailingMLAConfig
173+
cfg = BailingMLAConfig(hidden_size=hidden_size, **{
174+
k: v for k, v in params.items()
175+
if k in BailingMLAConfig.__dataclass_fields__
176+
})
177+
return BailingMLABlock(cfg)
178+
179+
180+
def _build_bailing_moe(hidden_size: int, params: dict) -> nn.Module:
181+
"""Ling-2.6-flash sparse MoE block (mlx-lm PR #1227)."""
182+
from cppmega_v4.nn.mlx_lm_bricks import BailingMoEBlock, BailingMoEConfig
183+
cfg = BailingMoEConfig(hidden_size=hidden_size, **{
184+
k: v for k, v in params.items()
185+
if k in BailingMoEConfig.__dataclass_fields__
186+
})
187+
return BailingMoEBlock(cfg)
188+
189+
140190
def _build_gated_attention(hidden_size: int, params: dict) -> nn.Module:
141191
"""Qwen3-Next / Qwen3.5 / Qwen3.6 Gated Attention.
142192
@@ -299,6 +349,17 @@ def __call__(self, x):
299349
"mla": _build_mla,
300350
"mla_absorb": _build_mla,
301351
"lightning_indexer": _build_lightning_indexer,
352+
# ----- bricks imported from open mlx-lm PRs (cppmega-integration branch) -----
353+
# mistral4_mla = Mistral Small 4 absorbed MLA + INT4 latent cache (PR #1037)
354+
# dsv4_attention = DeepSeek-V4 (Flash) hash-indexed sparse attention (PR #1201)
355+
# bailing_linear = Ling-2.6-flash linear attention (PR #1227)
356+
# bailing_mla = Ling-2.6-flash multi-latent attention (PR #1227)
357+
# bailing_moe = Ling-2.6-flash sparse MoE block (PR #1227)
358+
"mistral4_mla": _build_mistral4_mla,
359+
"dsv4_attention": _build_dsv4_attention,
360+
"bailing_linear": _build_bailing_linear,
361+
"bailing_mla": _build_bailing_mla,
362+
"bailing_moe": _build_bailing_moe,
302363
}
303364

304365

0 commit comments

Comments
 (0)