Align LazyTurtle HF key resolution with reversed WeightRenaming semantics (#2793)

* Align LazyTurtle HF key resolution with reversed WeightRenaming semantics

Signed-off-by: ZX-ModelCloud <zx@modelcloud.ai>

* cleanup

Signed-off-by: ZX-ModelCloud <zx@modelcloud.ai>

* removed "checkpoint_path_aliases"

Signed-off-by: ZX-ModelCloud <zx@modelcloud.ai>

---------

Signed-off-by: ZX-ModelCloud <zx@modelcloud.ai>

Author: ZX-ModelCloud

gptqmodel/models/base.py

@@ -247,12 +247,9 @@ class BaseQModel(nn.Module):
     server = None
 
     support_offload_to_disk = True
-    # Optional runtime->checkpoint prefix overrides used by LazyTurtle when the
-    # execution shell inserts wrapper modules that are absent from the checkpoint.
-    checkpoint_path_aliases: Optional[tuple[tuple[str, str], ...]] = None
-    # Optional runtime->checkpoint prefix overrides for LazyTurtle. When unset,
-    # the loader derives them from Hugging Face conversion mappings.
-    HF_CONVERSION_MAP_REVERSED: Optional[Dict[str, str]] = None
+    # Optional runtime->checkpoint overrides for LazyTurtle. Prefer reversed
+    # `WeightRenaming` entries; legacy runtime->checkpoint dicts are still accepted.
+    HF_CONVERSION_MAP_REVERSED: Optional[Any] = None
 
     moe_expert_module_name_prefixes = [".expert"]
 
@@ -423,7 +420,7 @@ def has_moe_flag(cls, module_spec: str) -> bool:
         return MOE_FLAG.lstrip(":") in flags
 
     @classmethod
-    def resolve_hf_conversion_map_reversed(cls, target_model: Optional[nn.Module] = None) -> Optional[Dict[str, str]]:
+    def resolve_hf_conversion_map_reversed(cls, target_model: Optional[nn.Module] = None) -> Optional[Any]:
         configured_map = getattr(cls, "HF_CONVERSION_MAP_REVERSED", None)
         if configured_map is not None:
             return copy.deepcopy(configured_map)

gptqmodel/models/definitions/base_qwen2_vl.py

@@ -21,14 +21,6 @@ class BaseQwen2VLGPTQ(BaseQModel):
     loader = AutoModelForImageTextToText
 
     pre_lm_head_norm_module = ["model.language_model.norm", "language_model.norm"]
-    HF_CONVERSION_MAP_REVERSED = {
-        "model.language_model": "model",
-        "language_model": "model",
-    }
-    checkpoint_path_aliases = (
-        ("model.language_model", "model"),
-        ("language_model", "model"),
-    )
 
     module_tree = [
         "model",

gptqmodel/models/loader.py

@@ -657,7 +657,6 @@ def skip(*args, **kwargs):
                     config=model.config,
                     model_init_kwargs=shell_model_init_kwargs,
                     module_tree=copy.deepcopy(getattr(cls, "module_tree", None)),
-                    checkpoint_path_aliases=copy.deepcopy(getattr(cls, "checkpoint_path_aliases", None)),
                     hf_conversion_map_reversed=copy.deepcopy(
                         cls.resolve_hf_conversion_map_reversed(target_model=model)
                     ),

gptqmodel/utils/structure.py

@@ -9,7 +9,10 @@
 from safetensors.torch import save_file
 from torch import nn
 
+from gptqmodel.models.definitions.gemma3 import Gemma3ForConditionalGenerationGPTQ
 from gptqmodel.models.definitions.mixtral import MixtralQModel
+from gptqmodel.models.definitions.qwen2_5_vl import Qwen2_5_VLQModel
+from gptqmodel.models.definitions.qwen2_vl import Qwen2VLQModel
 from gptqmodel.utils import structure as structure_module
 from gptqmodel.utils.structure import LazyTurtle
 
@@ -66,6 +69,18 @@ def __init__(self):
         self.config = SimpleNamespace(model_type="gemma3")
 
 
+class _Qwen2VLDummyModel(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.config = SimpleNamespace(model_type="qwen2_vl")
+
+
+class _Qwen2_5_VLDummyModel(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.config = SimpleNamespace(model_type="qwen2_5_vl")
+
+
 class _WeightRenamingStub:
     def __init__(self, source_pattern: str, target_pattern: str):
         self.source_patterns = [source_pattern]
@@ -82,6 +97,21 @@ def _gemma3_weight_renamings():
     ]
 
 
+def _qwen2_vl_weight_renamings():
+    return [
+        _WeightRenamingStub(
+            r"(?<!_)model(?!\.(language_model|visual))",
+            "model.language_model",
+        ),
+        _WeightRenamingStub(r"^visual", "model.visual"),
+    ]
+
+
+def _renaming_pairs(renamings) -> list[tuple[str, str]]:
+    assert renamings is not None
+    return [(entry.source_patterns[0], entry.target_patterns[0]) for entry in renamings]
+
+
 def _assert_gemma3_alias_resolution(turtle: LazyTurtle) -> None:
     assert turtle._resolve_checkpoint_module_path("model.language_model") == "language_model.model"
     assert turtle._resolve_checkpoint_module_path("model.vision_tower") == "vision_tower"
@@ -119,27 +149,58 @@ def _assert_gemma3_alias_resolution(turtle: LazyTurtle) -> None:
     assert turtle._resolve_checkpoint_tensor_name("lm_head", "weight") == "language_model.lm_head.weight"
 
 
+def _assert_qwen2_vl_alias_resolution(turtle: LazyTurtle) -> None:
+    assert turtle._resolve_checkpoint_module_path("model.language_model") == "model"
+    assert turtle._resolve_checkpoint_module_path("model.visual") == "visual"
+
+    assert (
+        turtle._resolve_checkpoint_tensor_name(
+            "model.language_model.layers.0.mlp",
+            "gate_proj.weight",
+        )
+        == "model.layers.0.mlp.gate_proj.weight"
+    )
+    assert (
+        turtle._resolve_checkpoint_tensor_name(
+            "wrapper.model.language_model.layers.0.mlp",
+            "gate_proj.weight",
+        )
+        == "model.layers.0.mlp.gate_proj.weight"
+    )
+    assert (
+        turtle._resolve_checkpoint_tensor_name(
+            "model.visual.blocks.0.attn",
+            "weight",
+        )
+        == "visual.blocks.0.attn.weight"
+    )
+
+
 def test_lazy_turtle_reverses_transformers_weight_renaming_list():
     reversed_map = LazyTurtle.reverse_hf_conversion_map(_gemma3_weight_renamings())
 
-    assert reversed_map == {
-        "model.language_model": "language_model.model",
-        "lm_head": "language_model.lm_head",
-        "model.vision_tower": "vision_tower",
-        "model.multi_modal_projector": "multi_modal_projector",
-    }
+    assert _renaming_pairs(reversed_map) == [
+        ("model.language_model", r"^language_model.model"),
+        ("lm_head", r"^language_model.lm_head"),
+        ("model.vision_tower", r"^vision_tower"),
+        ("model.multi_modal_projector", r"^multi_modal_projector"),
+    ]
 
 
 def test_lazy_turtle_runtime_to_checkpoint_alias_candidates_do_not_expand_infinitely(tmp_path):
+    reversed_map = LazyTurtle.reverse_hf_conversion_map(
+        {
+            "language_model.model": "language_model",
+            "language_model.lm_head": "lm_head",
+        }
+    )
+
     turtle = _build_lazy_turtle(
         tmp_path,
         {
             "language_model.model.layers.0.self_attn.q_proj.weight": torch.zeros(2, 2),
         },
-        hf_conversion_map_reversed={
-            "language_model": "language_model.model",
-            "lm_head": "language_model.lm_head",
-        },
+        hf_conversion_map_reversed=reversed_map,
     )
 
     assert turtle._runtime_to_checkpoint_alias_candidates("language_model.layers.0") == [
@@ -148,6 +209,22 @@ def test_lazy_turtle_runtime_to_checkpoint_alias_candidates_do_not_expand_infini
     ]
 
 
+def test_lazy_turtle_applies_reversed_weight_renamings_with_capturing_groups(tmp_path):
+    reversed_map = LazyTurtle.reverse_hf_conversion_map(
+        [_WeightRenamingStub(r"(.+)", r"timm_model.\1")]
+    )
+
+    turtle = _build_lazy_turtle(
+        tmp_path,
+        {
+            "backbone.conv.weight": torch.zeros(2, 2),
+        },
+        hf_conversion_map_reversed=reversed_map,
+    )
+
+    assert turtle._resolve_checkpoint_tensor_name("timm_model.backbone.conv", "weight") == "backbone.conv.weight"
+
+
 def test_lazy_turtle_uses_transformers_checkpoint_conversion_mapping_for_gemma3(tmp_path, monkeypatch):
     conversion_mapping_module = SimpleNamespace(
         get_checkpoint_conversion_mapping=lambda model_type: _gemma3_weight_renamings()
@@ -170,6 +247,53 @@ def test_lazy_turtle_uses_transformers_checkpoint_conversion_mapping_for_gemma3(
     _assert_gemma3_alias_resolution(turtle)
 
 
+def test_lazy_turtle_uses_transformers_checkpoint_conversion_mapping_for_qwen2_vl(tmp_path, monkeypatch):
+    conversion_mapping_module = SimpleNamespace(
+        get_checkpoint_conversion_mapping=lambda model_type: _qwen2_vl_weight_renamings()
+        if model_type == "qwen2_vl"
+        else None
+    )
+    monkeypatch.setattr(structure_module, "import_module", lambda name: conversion_mapping_module)
+
+    turtle = _build_lazy_turtle(
+        tmp_path,
+        {
+            "model.layers.0.mlp.gate_proj.weight": torch.zeros(2, 2),
+            "visual.blocks.0.attn.weight": torch.zeros(2, 2),
+        },
+        module_tree=Qwen2VLQModel.module_tree,
+        target_model=_Qwen2VLDummyModel(),
+    )
+
+    _assert_qwen2_vl_alias_resolution(turtle)
+
+
+def test_lazy_turtle_uses_transformers_checkpoint_conversion_mapping_for_qwen2_5_vl(tmp_path, monkeypatch):
+    observed_model_types: list[str] = []
+
+    def _get_checkpoint_conversion_mapping(model_type: str):
+        observed_model_types.append(model_type)
+        if model_type == "qwen2_5_vl":
+            return _qwen2_vl_weight_renamings()
+        return None
+
+    conversion_mapping_module = SimpleNamespace(get_checkpoint_conversion_mapping=_get_checkpoint_conversion_mapping)
+    monkeypatch.setattr(structure_module, "import_module", lambda name: conversion_mapping_module)
+
+    turtle = _build_lazy_turtle(
+        tmp_path,
+        {
+            "model.layers.0.mlp.gate_proj.weight": torch.zeros(2, 2),
+            "visual.blocks.0.attn.weight": torch.zeros(2, 2),
+        },
+        module_tree=Qwen2_5_VLQModel.module_tree,
+        target_model=_Qwen2_5_VLDummyModel(),
+    )
+
+    assert observed_model_types == ["qwen2_5_vl"]
+    _assert_qwen2_vl_alias_resolution(turtle)
+
+
 def test_lazy_turtle_falls_back_to_legacy_checkpoint_conversion_mapping(tmp_path, monkeypatch):
     def _raise_import_error(_name: str):
         raise ImportError("transformers.conversion_mapping is unavailable")
@@ -190,6 +314,45 @@ def _raise_import_error(_name: str):
     _assert_gemma3_alias_resolution(turtle)
 
 
+def test_base_qmodel_prefers_manual_hf_conversion_map_reversed(tmp_path, monkeypatch):
+    manual_renamings = LazyTurtle.reverse_hf_conversion_map(_gemma3_weight_renamings())
+    assert manual_renamings is not None
+    monkeypatch.setattr(
+        Gemma3ForConditionalGenerationGPTQ,
+        "HF_CONVERSION_MAP_REVERSED",
+        manual_renamings,
+        raising=False,
+    )
+
+    def _unexpected_import(_name: str):
+        raise AssertionError("manual HF_CONVERSION_MAP_REVERSED should bypass inferred transformers mappings")
+
+    monkeypatch.setattr(structure_module, "import_module", _unexpected_import)
+
+    resolved = Gemma3ForConditionalGenerationGPTQ.resolve_hf_conversion_map_reversed(target_model=_Gemma3DummyModel())
+    assert _renaming_pairs(resolved) == _renaming_pairs(manual_renamings)
+
+    resolved[0].source_patterns[0] = "mutated.runtime"
+    resolved_again = Gemma3ForConditionalGenerationGPTQ.resolve_hf_conversion_map_reversed(
+        target_model=_Gemma3DummyModel()
+    )
+    assert _renaming_pairs(resolved_again) == _renaming_pairs(manual_renamings)
+
+    turtle = _build_lazy_turtle(
+        tmp_path,
+        {
+            "language_model.model.layers.0.mlp.gate_proj.weight": torch.zeros(2, 2),
+            "vision_tower.vision_model.head.weight": torch.zeros(2, 2),
+            "multi_modal_projector.mm_input_projection_weight": torch.zeros(2, 2),
+            "language_model.lm_head.weight": torch.zeros(2, 2),
+        },
+        module_tree=Gemma3ForConditionalGenerationGPTQ.module_tree,
+        hf_conversion_map_reversed=resolved_again,
+    )
+
+    _assert_gemma3_alias_resolution(turtle)
+
+
 def test_lazy_turtle_keeps_module_tree_alias_resolution_for_mixtral(tmp_path):
     turtle = _build_lazy_turtle(
         tmp_path,

tests/test_lazy_turtle_conversion_mapping.py

@@ -1,3 +1,4 @@
+import copy
 import json
 from types import SimpleNamespace
 
@@ -496,16 +497,18 @@ class DummyQModel:
         require_pkgs = []
         supports_desc_act = [True, False]
         support_offload_to_disk = True
-        checkpoint_path_aliases = (("shell_model", "model"),)
+        HF_CONVERSION_MAP_REVERSED = (
+            SimpleNamespace(source_patterns=["shell_model"], target_patterns=["model"]),
+        )
         config_class = None
 
         @staticmethod
         def before_model_load(*_args, **_kwargs):
             return None
 
-        @staticmethod
-        def resolve_hf_conversion_map_reversed(*_args, **_kwargs):
-            return None
+        @classmethod
+        def resolve_hf_conversion_map_reversed(cls, *_args, **_kwargs):
+            return copy.deepcopy(cls.HF_CONVERSION_MAP_REVERSED)
 
         def __init__(self, model, **kwargs):
             self.model = model
@@ -531,7 +534,13 @@ def __init__(self, model, **kwargs):
     assert shell_configs
     assert load_calls == []
     assert isinstance(instance.turtle_model, LazyTurtle)
-    assert instance.turtle_model._runtime_to_checkpoint_aliases == DummyQModel.checkpoint_path_aliases
+    assert [
+        (entry.source_patterns[0], entry.target_patterns[0])
+        for entry in instance.turtle_model._runtime_to_checkpoint_renamings
+    ] == [
+        (entry.source_patterns[0], entry.target_patterns[0])
+        for entry in DummyQModel.HF_CONVERSION_MAP_REVERSED
+    ]
     assert instance.turtle_model.config._experts_implementation == "linear_loop"
     assert instance.turtle_model.config is not instance.model.config
 

tests/test_local_model_paths.py

@@ -623,9 +623,9 @@ def test_lazy_turtle_hf_conversion_map_reversed_resolves_nested_language_model_p
         model_local_path=str(model_dir),
         config=SimpleNamespace(_experts_implementation=None),
         model_init_kwargs={"device_map": {"": "cpu"}},
-        checkpoint_path_aliases=(
-            ("model.language_model", "model"),
-            ("language_model", "model"),
+        hf_conversion_map_reversed=(
+            SimpleNamespace(source_patterns=["model.language_model"], target_patterns=["model"]),
+            SimpleNamespace(source_patterns=["language_model"], target_patterns=["model"]),
         ),
     )
     assert source is not None

tests/test_offload_files.py

@@ -16,6 +16,7 @@
 from transformers import PreTrainedTokenizerFast
 
 from gptqmodel.models.definitions import base_qwen2_5_omni, base_qwen2_vl
+from gptqmodel.utils import structure as structure_module
 from gptqmodel.utils.hf import load_tokenizer
 
 
@@ -37,12 +38,26 @@ def test_qwen2_vl_image_only_process_vision_info_returns_image_list():
     assert image_inputs == [image]
 
 
-def test_qwen2_vl_declares_checkpoint_path_aliases_for_language_model_shell():
-    assert base_qwen2_vl.BaseQwen2VLGPTQ.support_offload_to_disk is True
-    assert base_qwen2_vl.BaseQwen2VLGPTQ.checkpoint_path_aliases == (
-        ("model.language_model", "model"),
-        ("language_model", "model"),
+def test_qwen2_vl_resolves_hf_conversion_map_reversed_for_language_model_shell(monkeypatch):
+    conversion_mapping_module = types.SimpleNamespace(
+        get_checkpoint_conversion_mapping=lambda model_type: [
+            types.SimpleNamespace(
+                source_patterns=[r"(?<!_)model(?!\.(language_model|visual))"],
+                target_patterns=["model.language_model"],
+            ),
+        ]
+        if model_type == "qwen2_vl"
+        else None
     )
+    monkeypatch.setattr(structure_module, "import_module", lambda name: conversion_mapping_module)
+
+    target_model = types.SimpleNamespace(config=types.SimpleNamespace(model_type="qwen2_vl"))
+    resolved = base_qwen2_vl.BaseQwen2VLGPTQ.resolve_hf_conversion_map_reversed(target_model=target_model)
+
+    assert base_qwen2_vl.BaseQwen2VLGPTQ.support_offload_to_disk is True
+    assert [(entry.source_patterns[0], entry.target_patterns[0]) for entry in resolved] == [
+        ("model.language_model", r"(?<!_)model(?!\.(language_model|visual))"),
+    ]
 
 
 def test_qwen2_vl_pre_quantize_hooks_use_inner_model_layout():