|
| 1 | +# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +from ..configuration_utils import PretrainedConfig |
| 16 | + |
| 17 | + |
| 18 | +class Gemma4MoeConfig(PretrainedConfig): |
| 19 | + """Gemma4 26B-A4B text-only MoE model configuration. |
| 20 | +
|
| 21 | + NOTE: Gemma4 currently only has VL checkpoints (Gemma4ForConditionalGeneration), |
| 22 | + no standalone text-only checkpoint exists. The config.json has top-level |
| 23 | + model_type="gemma4" (VL wrapper) with text params nested in text_config |
| 24 | + (model_type="gemma4_text"). |
| 25 | +
|
| 26 | + Both "gemma4" and "gemma4_text" are registered in CONFIG_MAPPING to this class, |
| 27 | + and from_dict() automatically extracts text_config from the VL wrapper. |
| 28 | +
|
| 29 | + TODO(VL): When implementing full multimodal Gemma4: |
| 30 | + 1. Create Gemma4Config (VL wrapper with text_config + vision_config) |
| 31 | + 2. Change CONFIG_MAPPING "gemma4" to point to Gemma4Config |
| 32 | + 3. Keep only "gemma4_text" -> Gemma4MoeConfig |
| 33 | + 4. Remove the text_config extraction logic in from_dict() below |
| 34 | + """ |
| 35 | + |
| 36 | + model_type = "gemma4_text" |
| 37 | + keys_to_ignore_at_inference = ["past_key_values"] |
| 38 | + |
| 39 | + @classmethod |
| 40 | + def from_dict(cls, config_dict, **kwargs): |
| 41 | + """Load text config from VL wrapper config. |
| 42 | +
|
| 43 | + Gemma4 only ships VL checkpoints, so config.json outer model_type="gemma4" |
| 44 | + with text params in text_config. This extracts it automatically. |
| 45 | +
|
| 46 | + TODO(VL): Move this extraction to Gemma4Config when VL model is implemented. |
| 47 | + """ |
| 48 | + if config_dict.get("model_type") == "gemma4" and "text_config" in config_dict: |
| 49 | + config_dict = config_dict["text_config"] |
| 50 | + return super().from_dict(config_dict, **kwargs) |
| 51 | + |
| 52 | + def __init__( |
| 53 | + self, |
| 54 | + vocab_size=262144, |
| 55 | + hidden_size=2816, |
| 56 | + intermediate_size=2112, |
| 57 | + moe_intermediate_size=704, |
| 58 | + num_hidden_layers=30, |
| 59 | + num_attention_heads=16, |
| 60 | + num_key_value_heads=8, |
| 61 | + num_global_key_value_heads=2, |
| 62 | + head_dim=256, |
| 63 | + global_head_dim=512, |
| 64 | + hidden_act="gelu_pytorch_tanh", |
| 65 | + hidden_activation=None, |
| 66 | + max_position_embeddings=262144, |
| 67 | + rms_norm_eps=1e-6, |
| 68 | + # RoPE |
| 69 | + rope_parameters=None, |
| 70 | + sliding_window_rope_base=10000.0, |
| 71 | + full_attention_rope_base=1000000.0, |
| 72 | + full_attention_rope_partial_factor=0.25, |
| 73 | + # Attention pattern |
| 74 | + layer_types=None, |
| 75 | + sliding_window=1024, |
| 76 | + interleaved_attn_pattern=(5, 1), |
| 77 | + # MoE |
| 78 | + num_experts=128, |
| 79 | + top_k_experts=8, |
| 80 | + scoring_func="sigmoid", |
| 81 | + enable_moe_block=True, |
| 82 | + # Gemma4-specific |
| 83 | + attention_k_eq_v=True, |
| 84 | + final_logit_softcapping=30.0, |
| 85 | + scale_embeddings_by_hidden_size=True, |
| 86 | + # Pipeline |
| 87 | + pp_seg_method="layer:Gemma4TransformerLayer", |
| 88 | + tie_word_embeddings=True, |
| 89 | + **kwargs, |
| 90 | + ): |
| 91 | + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) |
| 92 | + self.vocab_size = vocab_size |
| 93 | + self.hidden_size = hidden_size |
| 94 | + self.intermediate_size = intermediate_size |
| 95 | + self.moe_intermediate_size = moe_intermediate_size |
| 96 | + self.num_hidden_layers = num_hidden_layers |
| 97 | + self.num_attention_heads = num_attention_heads |
| 98 | + self.num_key_value_heads = num_key_value_heads |
| 99 | + self.num_global_key_value_heads = num_global_key_value_heads |
| 100 | + self.head_dim = head_dim |
| 101 | + self.global_head_dim = global_head_dim |
| 102 | + self.hidden_act = hidden_activation or hidden_act |
| 103 | + self.max_position_embeddings = max_position_embeddings |
| 104 | + self.rms_norm_eps = rms_norm_eps |
| 105 | + self.enable_moe_block = enable_moe_block |
| 106 | + self.sliding_window = sliding_window |
| 107 | + self.interleaved_attn_pattern = interleaved_attn_pattern |
| 108 | + self.num_experts = num_experts |
| 109 | + self.top_k_experts = top_k_experts |
| 110 | + self.scoring_func = scoring_func |
| 111 | + self.attention_k_eq_v = attention_k_eq_v |
| 112 | + self.final_logit_softcapping = final_logit_softcapping |
| 113 | + self.scale_embeddings_by_hidden_size = scale_embeddings_by_hidden_size |
| 114 | + self.pp_seg_method = pp_seg_method |
| 115 | + |
| 116 | + # Parse rope_parameters from HF config format |
| 117 | + if rope_parameters is not None: |
| 118 | + sliding_rope = rope_parameters.get("sliding_attention", {}) |
| 119 | + full_rope = rope_parameters.get("full_attention", {}) |
| 120 | + self.sliding_window_rope_base = sliding_rope.get("rope_theta", sliding_window_rope_base) |
| 121 | + self.full_attention_rope_base = full_rope.get("rope_theta", full_attention_rope_base) |
| 122 | + self.full_attention_rope_partial_factor = full_rope.get( |
| 123 | + "partial_rotary_factor", full_attention_rope_partial_factor |
| 124 | + ) |
| 125 | + else: |
| 126 | + self.sliding_window_rope_base = sliding_window_rope_base |
| 127 | + self.full_attention_rope_base = full_attention_rope_base |
| 128 | + self.full_attention_rope_partial_factor = full_attention_rope_partial_factor |
| 129 | + |
| 130 | + # Build layer_types from interleaved pattern if not provided |
| 131 | + if layer_types is None: |
| 132 | + sliding_count, global_count = self.interleaved_attn_pattern |
| 133 | + pattern = ["sliding_attention"] * sliding_count + ["full_attention"] * global_count |
| 134 | + self.layer_types = (pattern * ((num_hidden_layers // len(pattern)) + 1))[:num_hidden_layers] |
| 135 | + else: |
| 136 | + self.layer_types = layer_types |
0 commit comments