|
| 1 | +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +"""PaliGemma2 model configuration""" |
| 16 | + |
| 17 | +from typing import Optional, Union |
| 18 | + |
| 19 | +from paddleformers.transformers.configuration_utils import PretrainedConfig |
| 20 | + |
| 21 | + |
| 22 | +class SiglipVisionConfig(PretrainedConfig): |
| 23 | + """Configuration for the SigLIP vision encoder used in PaliGemma2.""" |
| 24 | + |
| 25 | + model_type = "siglip_vision_model" |
| 26 | + |
| 27 | + def __init__( |
| 28 | + self, |
| 29 | + hidden_size: int = 1152, |
| 30 | + image_size: int = 448, |
| 31 | + intermediate_size: int = 4304, |
| 32 | + num_hidden_layers: int = 27, |
| 33 | + num_attention_heads: int = 16, |
| 34 | + num_channels: int = 3, |
| 35 | + patch_size: int = 14, |
| 36 | + projection_dim: int = 3584, |
| 37 | + num_image_tokens: int = 1024, |
| 38 | + num_positions: int = 1024, |
| 39 | + layer_norm_eps: float = 1e-6, |
| 40 | + vision_use_head: bool = False, |
| 41 | + **kwargs, |
| 42 | + ): |
| 43 | + super().__init__(**kwargs) |
| 44 | + self.hidden_size = hidden_size |
| 45 | + self.image_size = image_size |
| 46 | + self.intermediate_size = intermediate_size |
| 47 | + self.num_hidden_layers = num_hidden_layers |
| 48 | + self.num_attention_heads = num_attention_heads |
| 49 | + self.num_channels = num_channels |
| 50 | + self.patch_size = patch_size |
| 51 | + self.projection_dim = projection_dim |
| 52 | + self.num_image_tokens = num_image_tokens |
| 53 | + self.num_positions = num_positions |
| 54 | + self.layer_norm_eps = layer_norm_eps |
| 55 | + self.vision_use_head = vision_use_head |
| 56 | + |
| 57 | + |
| 58 | +class Gemma2TextConfig(PretrainedConfig): |
| 59 | + """Configuration for the Gemma2 text decoder used in PaliGemma2.""" |
| 60 | + |
| 61 | + model_type = "gemma2" |
| 62 | + keys_to_ignore_at_inference = ["past_key_values"] |
| 63 | + |
| 64 | + def __init__( |
| 65 | + self, |
| 66 | + vocab_size: int = 257216, |
| 67 | + hidden_size: int = 3584, |
| 68 | + intermediate_size: int = 14336, |
| 69 | + num_hidden_layers: int = 42, |
| 70 | + num_attention_heads: int = 16, |
| 71 | + num_key_value_heads: int = 8, |
| 72 | + head_dim: Optional[int] = None, |
| 73 | + hidden_activation: str = "gelu_pytorch_tanh", |
| 74 | + max_position_embeddings: int = 8192, |
| 75 | + initializer_range: float = 0.02, |
| 76 | + rms_norm_eps: float = 1e-6, |
| 77 | + use_cache: bool = True, |
| 78 | + pad_token_id: int = 0, |
| 79 | + bos_token_id: int = 2, |
| 80 | + eos_token_id: int = 1, |
| 81 | + tie_word_embeddings: bool = True, |
| 82 | + rope_theta: float = 10000.0, |
| 83 | + attention_bias: bool = False, |
| 84 | + attention_dropout: float = 0.0, |
| 85 | + query_pre_attn_scalar: float = 256.0, |
| 86 | + sliding_window: int = 4096, |
| 87 | + final_logit_softcapping: float = 30.0, |
| 88 | + attn_logit_softcapping: float = 50.0, |
| 89 | + **kwargs, |
| 90 | + ): |
| 91 | + super().__init__( |
| 92 | + pad_token_id=pad_token_id, |
| 93 | + bos_token_id=bos_token_id, |
| 94 | + eos_token_id=eos_token_id, |
| 95 | + tie_word_embeddings=tie_word_embeddings, |
| 96 | + **kwargs, |
| 97 | + ) |
| 98 | + self.vocab_size = vocab_size |
| 99 | + self.hidden_size = hidden_size |
| 100 | + self.intermediate_size = intermediate_size |
| 101 | + self.num_hidden_layers = num_hidden_layers |
| 102 | + self.num_attention_heads = num_attention_heads |
| 103 | + self.num_key_value_heads = num_key_value_heads |
| 104 | + self.head_dim = 256 if head_dim is None else head_dim |
| 105 | + self.hidden_activation = hidden_activation |
| 106 | + self.max_position_embeddings = max_position_embeddings |
| 107 | + self.initializer_range = initializer_range |
| 108 | + self.rms_norm_eps = rms_norm_eps |
| 109 | + self.use_cache = use_cache |
| 110 | + self.rope_theta = rope_theta |
| 111 | + self.attention_bias = attention_bias |
| 112 | + self.attention_dropout = attention_dropout |
| 113 | + self.query_pre_attn_scalar = query_pre_attn_scalar |
| 114 | + self.sliding_window = sliding_window |
| 115 | + self.final_logit_softcapping = final_logit_softcapping |
| 116 | + self.attn_logit_softcapping = attn_logit_softcapping |
| 117 | + |
| 118 | + |
| 119 | +class PaliGemma2Config(PretrainedConfig): |
| 120 | + """Configuration for the PaliGemma2 multimodal model. |
| 121 | +
|
| 122 | + PaliGemma2 combines a SigLIP vision encoder with a Gemma2 text decoder. |
| 123 | +
|
| 124 | + Example: |
| 125 | + ```python |
| 126 | + >>> from paddleformers.transformers.paligemma2 import PaliGemma2Config |
| 127 | + >>> config = PaliGemma2Config() |
| 128 | + ``` |
| 129 | + """ |
| 130 | + |
| 131 | + model_type = "paligemma2" |
| 132 | + attribute_map = {"image_token_id": "image_token_index"} |
| 133 | + sub_configs = {"text_config": Gemma2TextConfig, "vision_config": SiglipVisionConfig} |
| 134 | + keys_to_ignore_at_inference = ["past_key_values"] |
| 135 | + |
| 136 | + def __init__( |
| 137 | + self, |
| 138 | + vision_config: Optional[Union[SiglipVisionConfig, dict]] = None, |
| 139 | + text_config: Optional[Union[Gemma2TextConfig, dict]] = None, |
| 140 | + image_token_index: int = 257152, |
| 141 | + projection_dim: int = 3584, |
| 142 | + hidden_size: int = 3584, |
| 143 | + vocab_size: int = 257152, |
| 144 | + tie_word_embeddings: bool = True, |
| 145 | + **kwargs, |
| 146 | + ): |
| 147 | + if vision_config is None: |
| 148 | + vision_config = SiglipVisionConfig() |
| 149 | + elif isinstance(vision_config, dict): |
| 150 | + vision_config = SiglipVisionConfig(**vision_config) |
| 151 | + |
| 152 | + if text_config is None: |
| 153 | + text_config = Gemma2TextConfig() |
| 154 | + elif isinstance(text_config, dict): |
| 155 | + text_config = Gemma2TextConfig(**text_config) |
| 156 | + |
| 157 | + self.vision_config = vision_config |
| 158 | + self.text_config = text_config |
| 159 | + self.image_token_index = image_token_index |
| 160 | + self.projection_dim = projection_dim |
| 161 | + self.hidden_size = hidden_size |
| 162 | + self.vocab_size = vocab_size |
| 163 | + self.tie_word_embeddings = tie_word_embeddings |
| 164 | + |
| 165 | + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) |
| 166 | + |
| 167 | + |
| 168 | +__all__ = ["PaliGemma2Config", "SiglipVisionConfig", "Gemma2TextConfig"] |
0 commit comments