diff --git a/src/peft/tuners/tuners_utils.py b/src/peft/tuners/tuners_utils.py index 02e6f2feb7..5abaac5e7f 100644 --- a/src/peft/tuners/tuners_utils.py +++ b/src/peft/tuners/tuners_utils.py @@ -1722,7 +1722,10 @@ def check_target_module_exists(config, key: str) -> bool | re.Match[str] | None: # TODO: It's still unclear how empty layers_pattern (None, [], or "") should behave # For now, empty layers_pattern means any layer pattern is ok if layers_pattern is None or len(layers_pattern) == 0: - layer_index = re.match(r".*\.[^.]*\.(\d+)\.", key) + # Use non-greedy .*? to match the FIRST digit group, not the last + # This prevents MoE expert indices from being matched instead of layer indices + # e.g., for "model.layers.1.mlp.experts.0.up_proj", we want to match "1" not "0" + layer_index = re.match(r".*?\.[^.]*\.(\d+)\.", key) else: layers_pattern = [layers_pattern] if isinstance(layers_pattern, str) else layers_pattern for pattern in layers_pattern: diff --git a/tests/test_tuners_utils.py b/tests/test_tuners_utils.py index e93f87610b..d946d12627 100644 --- a/tests/test_tuners_utils.py +++ b/tests/test_tuners_utils.py @@ -128,6 +128,16 @@ ("blocks.1.bias", ["weight"], [1], ["blocks"], False), ("mlp.blocks.1.weight", ["weight"], [1], ["blocks"], True), ("mlp.blocks.1.bias", ["weight"], [1], ["blocks"], False), + # MoE models: layers_to_transform should match layer index, not expert index + # For "model.layers.1.mlp.experts.0.up_proj", should match layer 1, not expert 0 + ("model.layers.1.mlp.experts.0.up_proj", ["up_proj"], [1], ["layers"], True), + ("model.layers.1.mlp.experts.0.up_proj", ["up_proj"], [0], ["layers"], False), # expert 0, but layer 1 + ("model.layers.1.mlp.experts.1.up_proj", ["up_proj"], [1], ["layers"], True), # expert 1, layer 1 + ("model.layers.2.mlp.experts.1.up_proj", ["up_proj"], [1], ["layers"], False), # layer 2, not 1 + # MoE without explicit layers_pattern - should still match layer index, not expert index + ("model.layers.1.mlp.experts.0.up_proj", ["up_proj"], [1], None, True), + ("model.layers.1.mlp.experts.0.up_proj", ["up_proj"], [0], None, False), # must not match expert 0 + ("model.layers.2.mlp.experts.1.up_proj", ["up_proj"], [1], None, False), # must not match expert 1 ] MAYBE_INCLUDE_ALL_LINEAR_LAYERS_TEST_CASES = [