@@ -2865,8 +2865,12 @@ def __init__(self, *args, **kwargs):
28652865 # fix for SmolVLM2, missing `num_attention_heads` in config.json
28662866 if self.hf_arch == "VLlama3ForCausalLM":
28672867 self.hparams["num_attention_heads"] = self.hparams.get("num_attention_heads", 32)
2868- hparams = ModelBase.load_hparams(self.dir_model, is_mistral_format=False)
2869- self.origin_hf_arch = hparams.get('architectures', [None])[0]
2868+ # Mistral consolidated format has no config.json; origin_hf_arch is HF-only.
2869+ if self.is_mistral_format:
2870+ self.origin_hf_arch = None
2871+ else:
2872+ hparams = ModelBase.load_hparams(self.dir_model, is_mistral_format=False)
2873+ self.origin_hf_arch = hparams.get('architectures', [None])[0]
28702874
28712875 def set_vocab(self):
28722876 if self.origin_hf_arch == "GlmasrModel":
@@ -9760,6 +9764,73 @@ def prepare_tensors(self):
97609764 raise ValueError(f"Unprocessed experts: {experts}")
97619765
97629766
9767+ @ModelBase.register("MiMoV2ForCausalLM")
9768+ class MiMoV2VisionModel(MmprojModel):
9769+ def __init__(self, *args, **kwargs):
9770+ super().__init__(*args, **kwargs)
9771+ assert self.hparams_vision is not None
9772+ hp = self.hparams_vision
9773+
9774+ hp["image_size"] = hp.get("image_size", 560)
9775+ hp["num_attention_heads"] = hp.get("num_heads", 32)
9776+ hp["num_hidden_layers"] = hp.get("depth", 28)
9777+
9778+ self.n_q_heads = int(hp["num_heads"])
9779+ self.num_kv_heads = int(hp.get("num_key_value_heads", 8))
9780+ self.head_dim = int(hp.get("qk_channels", 64))
9781+ self.spatial_merge_size = int(hp["spatial_merge_size"])
9782+ # MiMoV2 vision RMSNorm: HF uses getattr(config, "rms_norm_eps", 1e-6) and the
9783+ # field is absent from MiMo-V2.5's vision_config
9784+ self.rms_norm_eps = float(hp.get("rms_norm_eps", 1e-6))
9785+
9786+ # fullatt_block_indexes are also reflected in vit_window_attn_types as -1
9787+ self.fullatt_block_indexes = list(hp.get("fullatt_block_indexes") or [])
9788+ self.vit_window_attn_types = list(hp.get("vit_window_attn_types") or [])
9789+ self.visual_token_window_size = int(hp.get("visual_token_window_size", -1))
9790+ self.use_sink = bool(hp.get("use_sink", False))
9791+
9792+ def set_gguf_parameters(self):
9793+ super().set_gguf_parameters()
9794+
9795+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MIMOVL)
9796+ self.gguf_writer.add_vision_use_silu(True)
9797+ self.gguf_writer.add_vision_head_count_kv(self.num_kv_heads)
9798+ self.gguf_writer.add_vision_spatial_merge_size(self.spatial_merge_size)
9799+ self.gguf_writer.add_uint32(gguf.Keys.ClipVision.WINDOW_SIZE, self.visual_token_window_size)
9800+ self.gguf_writer.add_vision_wa_pattern_mode(self.vit_window_attn_types)
9801+ self.gguf_writer.add_vision_attention_layernorm_eps(self.rms_norm_eps)
9802+ self.gguf_writer.add_vision_min_pixels(int(self.preprocessor_config["min_pixels"]))
9803+ self.gguf_writer.add_vision_max_pixels(int(self.preprocessor_config["max_pixels"]))
9804+
9805+ def tensor_force_quant(self, name, new_name, bid, n_dims):
9806+ # Sinks must be F32: any sink-style softmax/mask add in ggml requires
9807+ # F32, and we fold sinks into a host-built F32 mask at encode time.
9808+ if new_name.endswith(".attn_sinks"):
9809+ return gguf.GGMLQuantizationType.F32
9810+ return super().tensor_force_quant(name, new_name, bid, n_dims)
9811+
9812+ @classmethod
9813+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
9814+ name, _ = item
9815+ if not name.startswith("visual."):
9816+ return None
9817+ return super().filter_tensors(item)
9818+
9819+ def modify_tensors(self, data_torch, name, bid):
9820+ # Conv3D patch embed: split along the temporal axis (kt=2) into two Conv2D
9821+ # weights that the existing qwen2vl-style two-Conv2D path consumes.
9822+ if name == "visual.patch_embed.proj.weight":
9823+ _, _, kt, _, _ = data_torch.shape
9824+ if kt != 2:
9825+ raise ValueError(f"unexpected temporal_patch_size: {kt}")
9826+ embd_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH]
9827+ yield (embd_name + ".weight", data_torch[:, :, 0, ...])
9828+ yield (embd_name + ".weight.1", data_torch[:, :, 1, ...])
9829+ return
9830+
9831+ yield from super().modify_tensors(data_torch, name, bid)
9832+
9833+
97639834@ModelBase.register("Step3p5ForCausalLM")
97649835class Step35Model(TextModel):
97659836 model_arch = gguf.MODEL_ARCH.STEP35
@@ -13342,16 +13413,20 @@ def set_gguf_parameters(self):
1334213413 self.gguf_writer.add_vision_use_silu(True)
1334313414
1334413415 # spatial_merge_size
13345- if self.find_vparam(["mm_projector_id"]) == "patch_merge":
13416+ if self.find_vparam(["mm_projector_id"], optional=True ) == "patch_merge":
1334613417 self.gguf_writer.add_vision_spatial_merge_size(
1334713418 self.find_vparam(["spatial_merge_size"])
1334813419 )
1334913420
1335013421 def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:
1335113422 if name == "vision_language_adapter.w_in.weight":
1335213423 return "mm.1.weight"
13424+ elif name == "vision_language_adapter.w_in.bias":
13425+ return "mm.1.bias"
1335313426 elif name == "vision_language_adapter.w_out.weight":
1335413427 return "mm.2.weight"
13428+ elif name == "vision_language_adapter.w_out.bias":
13429+ return "mm.2.bias"
1335513430 return super().map_tensor_name(name, try_suffixes)
1335613431
1335713432
0 commit comments