[Cherry-Pick] PR 3583 into release/v1.0 (#3598)

Author: qhpeklh5959

paddleformers/cli/train/sft/workflow.py

@@ -235,6 +235,11 @@ def run_sft(
     if "DeepseekV3" in str(model_config.architectures):
         training_args.prediction_loss_only = True
 
+    if "qwen3_vl" in model_config.model_type and not model_args.lora:
+        if training_args.sequence_parallel:
+            logger.warning("Qwen3VL model do not support `sequence_parallel` yet, temporarily set to False")
+        training_args.sequence_parallel = False
+
     LlmMetaConfig.set_llm_config(model_config, training_args)
     model_config.use_fast_layer_norm = model_args.use_fast_layer_norm
 

paddleformers/transformers/qwen3_vl/modeling_fleet.py

@@ -228,7 +228,7 @@ def _forward_impl(
             packed_seq_params=packed_seq_params,
         )
         hidden_states = self._forward_mlp(hidden_states)
-        if self.layer_number in range(len(deepstack_visual_emb)):
+        if deepstack_visual_emb and self.layer_number in range(len(deepstack_visual_emb)):
             # print("process _deepstack_process ",hidden_states.shape,visual_pos_masks.shape,deepstack_visual_emb[self.layer_number].shape)
             hidden_states = self._deepstack_process(
                 hidden_states=hidden_states,
@@ -339,6 +339,7 @@ class Qwen3VLTextProvider(GPTModelProvider):
     use_flash_attention: bool = True
     use_fused_linear_cross_entropy: bool = True
     high_precision_rope: bool = True
+    moe_grouped_gemm: bool = True
 
     n_shared_experts: int = 0
     transform_rules = {
@@ -1125,15 +1126,6 @@ def forward(
         else:
             if position_ids.shape == input_ids.shape:
                 position_ids = position_ids.expand(3, position_ids.shape[0], -1)
-            else:
-                batch_size, seq_length = input_ids.shape
-                position_ids = paddle.arange(seq_length)
-                position_ids = position_ids.view(1, 1, -1).expand(3, batch_size, -1)
-                if cache_position is not None:
-                    delta = cache_position[0] + self.rope_deltas
-                else:
-                    delta = paddle.zeros((batch_size, seq_length))
-                position_ids = position_ids + delta
 
         input_dict = {
             "input_ids": input_ids,

paddleformers/transformers/qwen3_vl_moe/modeling.py

@@ -46,7 +46,7 @@
 from ..model_outputs import BaseModelOutputWithPast, ModelOutput
 from ..model_utils import PretrainedModel
 from ..modeling_rope_utils import ROPE_INIT_FUNCTIONS
-from ..qwen3_vl.modeling_fleet import Qwen3VLModel, Qwen3VLProvider
+from ..qwen3_vl.modeling_fleet import Qwen3VLModelDist, Qwen3VLProvider
 from ..utils import logger
 from .configuration import (
     Qwen3VLMoeConfig,
@@ -376,7 +376,7 @@ def _gen_aoa_config(cls, config: Qwen3VLMoeConfig):
             else:
                 split_experts_up_gate = ""
                 split_experts_down = ""
-                for expert_id in range(config.text_config.n_routed_experts):
+                for expert_id in range(config.text_config.num_experts):
                     split_experts_up_gate += f"{llm_prefix}{layer_id + 1}.mlp.experts.{expert_id}.up_gate_proj.weight,"
                     split_experts_down += f"{llm_prefix}{layer_id + 1}.mlp.experts.{expert_id}.down_proj.weight,"
                 split_experts_down += "axis=0"
@@ -2594,12 +2594,13 @@ def __new__(cls, config, have_criterion=True):
         config.pipeline_model_parallel_size = max(config.pipeline_model_parallel_size, 1)
         config.virtual_pipeline_model_parallel_size = max(config.virtual_pipeline_model_parallel_size, 1)
         config.expert_model_parallel_size = max(config.expert_model_parallel_size, 1)
+        config.moe_grouped_gemm = True
         criterion = None
         if have_criterion:
             criterion = CriterionLayer(config.text_config)
         model_provider_class = Qwen3VLProvider
         model_provider = model_provider_class.from_config(config)
-        qwen3vl_model = Qwen3VLModel(model_provider, model_version=config.model_type, criterion=criterion)
+        qwen3vl_model = Qwen3VLModelDist(model_provider, model_version=config.model_type, criterion=criterion)
         qwen3vl_model._gen_aoa_config = cls._gen_aoa_config
         qwen3vl_model._gen_inv_aoa_config = cls._gen_inv_aoa_config
         qwen3vl_model._get_tensor_parallel_mappings = cls._get_tensor_parallel_mappings