Add InternLM model and tokenizer support

paddleformers/cli/utils/llm_utils.py

@@ -77,7 +77,7 @@ def get_lora_target_modules(model):
         ]
     elif model.config.model_type == "bloom":
         target_modules = [".*query_key_value.*", ".*dense.*", ".*dense_h_to_4h.*", ".*dense_4h_to_h.*"]
-    elif model.config.model_type in ["llama", "jamba"]:
+    elif model.config.model_type in ["llama", "jamba", "internlm"]:
         target_modules = [
             ".*q_proj.*",
             ".*v_proj.*",

paddleformers/transformers/__init__.py

@@ -146,6 +146,13 @@
         "Ernie4_5ForCausalLMPipe",
     ],
     "ernie4_5.tokenizer": ["Ernie4_5Tokenizer"],
+    "internlm.configuration": ["InternLMConfig"],
+    "internlm.modeling": [
+        "InternLMModel",
+        "InternLMForCausalLM",
+        "InternLMForCausalLMPipe",
+    ],
+    "internlm.tokenizer": ["InternLMTokenizer"],
     "ernie4_5_moe.configuration": ["Ernie4_5_MoeConfig"],
     "ernie4_5_moe.modeling": ["Ernie4_5_MoeModel", "Ernie4_5_MoeForCausalLM", "Ernie4_5_MoeForCausalLMPipe"],
     "ernie4_5_moe_vl.configuration": ["Ernie4_5_VLConfig"],
@@ -418,6 +425,7 @@
     from .phi3 import *
     from .gemma3_text import *
     from .glm_ocr import *
+    from .internlm import *
 else:
     sys.modules[__name__] = _LazyModule(
         __name__,

paddleformers/transformers/auto/configuration.py

@@ -63,6 +63,7 @@
         ("glm_ocr", "GlmOcrConfig"),
         ("qwen3_5", "Qwen3_5Config"),
         ("qwen3_5_moe", "Qwen3_5MoEConfig"),
+        ("internlm", "InternLMConfig"),
     ]
 )
 
@@ -91,6 +92,7 @@
         ("glm_ocr", "GlmOcrForConditionalGeneration"),
         ("qwen3_5_moe", "Qwen3_5MoEForConditionalGeneration"),
         ("qwen3_5", "Qwen3_5ForConditionalGeneration"),
+        ("internlm", "InternLMForCausalLM"),
     ]
 )
 

paddleformers/transformers/auto/modeling.py

@@ -80,6 +80,7 @@
         ("Gemma3", "gemma3_text"),
         ("Glm4vMoe", "glm4v_moe"),
         ("GlmOcr", "glm_ocr"),
+        ("InternLM", "internlm"),
     ]
 )
 

paddleformers/transformers/internlm/__init__.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Package"""
+
+import sys
+from typing import TYPE_CHECKING
+
+from ...utils.lazy_import import _LazyModule
+
+import_structure = {
+    "tokenizer": ["InternLMTokenizer"],
+    "configuration": ["InternLMConfig"],
+    "modeling": [
+        "InternLMDecoderLayer",
+        "InternLMModel",
+        "InternLMForCausalLM",
+        "InternLMForCausalLMPipe",
+    ],
+}
+
+if TYPE_CHECKING:
+    from .configuration import *
+    from .modeling import *
+else:
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()["__file__"],
+        import_structure,
+        module_spec=__spec__,
+    )

paddleformers/transformers/internlm/auto_dist_config.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle.distributed as dist
+
+
+def get_dist_config(model, prefix=""):
+    """Generate distributed configuration for InternLM model"""
+    if prefix != "":
+        assert prefix.endswith(".")
+
+    config = {
+        "mp_config": {
+            "parallelize_plan": {
+                f"{prefix}model.embed_tokens": dist.ColWiseParallel(gather_output=True),
+                f"{prefix}model.layers.*.self_attn.q_proj": dist.ColWiseParallel(),
+                f"{prefix}model.layers.*.self_attn.k_proj": dist.ColWiseParallel(),
+                f"{prefix}model.layers.*.self_attn.v_proj": dist.ColWiseParallel(),
+                f"{prefix}model.layers.*.self_attn.o_proj": dist.RowWiseParallel(),
+                f"{prefix}model.layers.*.mlp.gate_proj": dist.ColWiseParallel(),
+                f"{prefix}model.layers.*.mlp.up_proj": dist.ColWiseParallel(),
+                f"{prefix}model.layers.*.mlp.down_proj": dist.RowWiseParallel(),
+                f"{prefix}lm_head.weight": dist.ColWiseParallel(),
+            }
+        },
+    }
+    return config

paddleformers/transformers/internlm/configuration.py

@@ -0,0 +1,126 @@
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""InternLM model configuration."""
+
+from ..configuration_utils import PretrainedConfig
+
+
+class InternLMConfig(PretrainedConfig):
+    model_type = "internlm"
+
+    def __init__(
+        self,
+        vocab_size=103168,
+        hidden_size=4096,
+        intermediate_size=11008,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        hidden_act="silu",
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        recompute_use_reentrant=False,
+        pad_token_id=0,
+        bos_token_id=1,
+        eos_token_id=2,
+        tie_word_embeddings=False,
+        bias=True,
+        rotary=None,
+        attention_dropout=0.0,
+        ignored_index=-100,
+        micro_batch_size=-1,
+        pp_seg_method="layer:InternLMDecoderLayer|EmptyLayer",
+        dpo_config=None,
+        kto_config=None,
+        recompute_granularity=None,
+        recompute_method=None,
+        recompute_modules=None,
+        recompute_num_layers=None,
+        recompute_mtp_granularity=None,
+        recompute_mtp_method=None,
+        recompute_mtp_modules=None,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_attention_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.bias = bias
+        self.attention_bias = bias
+        self.mlp_bias = False
+        self.attention_dropout = attention_dropout
+        self.head_dim = self.hidden_size // self.num_attention_heads
+
+        rotary = {"base": 10000, "type": "dynamic"} if rotary is None else rotary
+        self.rotary = rotary
+        self.rope_theta = rotary.get("base", 10000)
+        rotary_type = rotary.get("type", "dynamic")
+        if rotary_type == "origin":
+            self.rope_parameters = {"rope_theta": self.rope_theta, "rope_type": "default"}
+        elif rotary_type == "dynamic":
+            self.rope_parameters = {
+                "rope_theta": self.rope_theta,
+                "rope_type": "dynamic",
+                "factor": rotary.get("scaling_factor", 1.0),
+            }
+        else:
+            raise ValueError("InternLM only supports rotary type 'origin' or 'dynamic'.")
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+        self.recompute_use_reentrant = recompute_use_reentrant
+        self.ignored_index = ignored_index
+        self.micro_batch_size = micro_batch_size
+        self.pp_seg_method = pp_seg_method
+        self.dpo_config = dpo_config
+        self.kto_config = kto_config
+        self.recompute_granularity = recompute_granularity
+        self.recompute_method = recompute_method
+        self.recompute_modules = recompute_modules
+        self.recompute_num_layers = recompute_num_layers
+        self.recompute_mtp_granularity = recompute_mtp_granularity
+        self.recompute_mtp_method = recompute_mtp_method
+        self.recompute_mtp_modules = recompute_mtp_modules
+        self.register_unsavable_keys(
+            [
+                "ignored_index",
+                "recompute_use_reentrant",
+                "pp_seg_method",
+                "micro_batch_size",
+                "dpo_config",
+                "kto_config",
+                "recompute_granularity",
+                "recompute_method",
+                "recompute_modules",
+                "recompute_num_layers",
+                "recompute_mtp_granularity",
+                "recompute_mtp_method",
+                "recompute_mtp_modules",
+            ]
+        )