From 8308d99ca51ec3203d8bbe1e8d748ee6a557956e Mon Sep 17 00:00:00 2001 From: Severus Qin Date: Fri, 16 Jan 2026 21:27:07 +0800 Subject: [PATCH] Cherry-Pick PR 3583 into release/v1.0 --- paddleformers/cli/train/sft/workflow.py | 5 +++++ .../transformers/qwen3_vl/modeling_fleet.py | 12 ++---------- paddleformers/transformers/qwen3_vl_moe/modeling.py | 7 ++++--- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/paddleformers/cli/train/sft/workflow.py b/paddleformers/cli/train/sft/workflow.py index 6cdc8e76ad0..b3e8e1d2599 100644 --- a/paddleformers/cli/train/sft/workflow.py +++ b/paddleformers/cli/train/sft/workflow.py @@ -235,6 +235,11 @@ def run_sft( if "DeepseekV3" in str(model_config.architectures): training_args.prediction_loss_only = True + if "qwen3_vl" in model_config.model_type and not model_args.lora: + if training_args.sequence_parallel: + logger.warning("Qwen3VL model do not support `sequence_parallel` yet, temporarily set to False") + training_args.sequence_parallel = False + LlmMetaConfig.set_llm_config(model_config, training_args) model_config.use_fast_layer_norm = model_args.use_fast_layer_norm diff --git a/paddleformers/transformers/qwen3_vl/modeling_fleet.py b/paddleformers/transformers/qwen3_vl/modeling_fleet.py index 906969baf5d..a156b10e924 100644 --- a/paddleformers/transformers/qwen3_vl/modeling_fleet.py +++ b/paddleformers/transformers/qwen3_vl/modeling_fleet.py @@ -228,7 +228,7 @@ def _forward_impl( packed_seq_params=packed_seq_params, ) hidden_states = self._forward_mlp(hidden_states) - if self.layer_number in range(len(deepstack_visual_emb)): + if deepstack_visual_emb and self.layer_number in range(len(deepstack_visual_emb)): # print("process _deepstack_process ",hidden_states.shape,visual_pos_masks.shape,deepstack_visual_emb[self.layer_number].shape) hidden_states = self._deepstack_process( hidden_states=hidden_states, @@ -339,6 +339,7 @@ class Qwen3VLTextProvider(GPTModelProvider): use_flash_attention: bool = True use_fused_linear_cross_entropy: bool = True high_precision_rope: bool = True + moe_grouped_gemm: bool = True n_shared_experts: int = 0 transform_rules = { @@ -1125,15 +1126,6 @@ def forward( else: if position_ids.shape == input_ids.shape: position_ids = position_ids.expand(3, position_ids.shape[0], -1) - else: - batch_size, seq_length = input_ids.shape - position_ids = paddle.arange(seq_length) - position_ids = position_ids.view(1, 1, -1).expand(3, batch_size, -1) - if cache_position is not None: - delta = cache_position[0] + self.rope_deltas - else: - delta = paddle.zeros((batch_size, seq_length)) - position_ids = position_ids + delta input_dict = { "input_ids": input_ids, diff --git a/paddleformers/transformers/qwen3_vl_moe/modeling.py b/paddleformers/transformers/qwen3_vl_moe/modeling.py index 3c3ad621adc..32e57a562d0 100644 --- a/paddleformers/transformers/qwen3_vl_moe/modeling.py +++ b/paddleformers/transformers/qwen3_vl_moe/modeling.py @@ -46,7 +46,7 @@ from ..model_outputs import BaseModelOutputWithPast, ModelOutput from ..model_utils import PretrainedModel from ..modeling_rope_utils import ROPE_INIT_FUNCTIONS -from ..qwen3_vl.modeling_fleet import Qwen3VLModel, Qwen3VLProvider +from ..qwen3_vl.modeling_fleet import Qwen3VLModelDist, Qwen3VLProvider from ..utils import logger from .configuration import ( Qwen3VLMoeConfig, @@ -376,7 +376,7 @@ def _gen_aoa_config(cls, config: Qwen3VLMoeConfig): else: split_experts_up_gate = "" split_experts_down = "" - for expert_id in range(config.text_config.n_routed_experts): + for expert_id in range(config.text_config.num_experts): split_experts_up_gate += f"{llm_prefix}{layer_id + 1}.mlp.experts.{expert_id}.up_gate_proj.weight," split_experts_down += f"{llm_prefix}{layer_id + 1}.mlp.experts.{expert_id}.down_proj.weight," split_experts_down += "axis=0" @@ -2594,12 +2594,13 @@ def __new__(cls, config, have_criterion=True): config.pipeline_model_parallel_size = max(config.pipeline_model_parallel_size, 1) config.virtual_pipeline_model_parallel_size = max(config.virtual_pipeline_model_parallel_size, 1) config.expert_model_parallel_size = max(config.expert_model_parallel_size, 1) + config.moe_grouped_gemm = True criterion = None if have_criterion: criterion = CriterionLayer(config.text_config) model_provider_class = Qwen3VLProvider model_provider = model_provider_class.from_config(config) - qwen3vl_model = Qwen3VLModel(model_provider, model_version=config.model_type, criterion=criterion) + qwen3vl_model = Qwen3VLModelDist(model_provider, model_version=config.model_type, criterion=criterion) qwen3vl_model._gen_aoa_config = cls._gen_aoa_config qwen3vl_model._gen_inv_aoa_config = cls._gen_inv_aoa_config qwen3vl_model._get_tensor_parallel_mappings = cls._get_tensor_parallel_mappings