From e7e19079dca4465e04cb1a0a8a59e5cbf89ede42 Mon Sep 17 00:00:00 2001
From: zhengshengning <ningzhengsheng@baidu.com>
Date: Thu, 18 Jun 2026 23:15:02 +0800
Subject: [PATCH 1/7] fix sequentialMLP

---
 megatron/core/transformer/moe/experts.py | 29 ++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py
index 34e9fb17a02..ddff045033b 100644
--- a/megatron/core/transformer/moe/experts.py
+++ b/megatron/core/transformer/moe/experts.py
@@ -724,6 +724,28 @@ def forward(
             return super().forward(permuted_local_hidden_states, tokens_per_expert, permuted_probs)
 
 
+class _SeqMLPProxy:
+    """Expose GroupedMLP-compatible ``weight{i}`` / ``bias{i}`` access on top of
+    ``SequentialMLP.local_experts[i].linear_fc{1,2}``.
+
+    Required by upper layers (e.g. ms-swift's ``GPTBridge._set_mlp_state``) that
+    probe ``mg_mlp.linear_fc1`` / ``linear_fc2`` like GroupedMLP.
+    """
+
+    def __init__(self, experts, attr):
+        self._experts = experts
+        self._attr = attr
+
+    def __getattr__(self, name):
+        if name.startswith('weight'):
+            idx = int(name[len('weight'):])
+            return getattr(self._experts[idx], self._attr).weight
+        if name.startswith('bias'):
+            idx = int(name[len('bias'):])
+            return getattr(self._experts[idx], self._attr).bias
+        raise AttributeError(name)
+
+
 class SequentialMLP(MegatronModule):
     """An implementation of the Experts layer using a sequence of MLP layers.
 
@@ -766,6 +788,13 @@ def __init__(
             )
             self.local_experts.append(expert)
 
+        # ---- alignment patch: expose GroupedMLP-style interfaces on SequentialMLP ----
+        # Upper layers (e.g. ms-swift gpt_bridge._set_mlp_state) probe
+        # `mg_mlp.linear_fc1` / `linear_fc2` and expect `weight{i}`/`bias{i}` access
+        # like GroupedMLP. Inject a thin proxy that forwards to local_experts[i].
+        self.linear_fc1 = _SeqMLPProxy(self.local_experts, 'linear_fc1')
+        self.linear_fc2 = _SeqMLPProxy(self.local_experts, 'linear_fc2')
+
     def _pad_tensor_for_quantization(self, hidden, probs):
         """Padding tensor shape to multiples of 16/32."""
         actual_num_tokens = hidden.shape[0]

From 3079b145f57fde6a45c7640093127e284984c2ab Mon Sep 17 00:00:00 2001
From: zhengshengning <ningzhengsheng@baidu.com>
Date: Wed, 24 Jun 2026 14:21:59 +0800
Subject: [PATCH 2/7] align: preserve fp32 router expert bias

---
 megatron/core/transformer/module.py | 44 +++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/megatron/core/transformer/module.py b/megatron/core/transformer/module.py
index 6539ee36105..41c924fb8a7 100644
--- a/megatron/core/transformer/module.py
+++ b/megatron/core/transformer/module.py
@@ -23,6 +23,40 @@
 _BF16_TYPES = (torch.BFloat16Tensor, torch.cuda.BFloat16Tensor)
 
 
+def _iter_modules_with_expert_bias(module):
+    if module is None:
+        return
+    for submodule in module.modules():
+        expert_bias = getattr(submodule, 'expert_bias', None)
+        if expert_bias is not None:
+            yield submodule, expert_bias
+
+
+def _snapshot_expert_bias_fp32(module):
+    snapshots = {}
+    for submodule, expert_bias in _iter_modules_with_expert_bias(module):
+        snapshot = expert_bias.detach().clone().to(
+            device=expert_bias.device,
+            dtype=torch.float32,
+        )
+        snapshots[submodule] = snapshot
+        submodule._run_torch_master_expert_bias_fp32 = snapshot
+    return snapshots
+
+
+def _restore_expert_bias_fp32(module, snapshots):
+    restored = 0
+    for submodule, expert_bias in _iter_modules_with_expert_bias(module):
+        snapshot = snapshots.get(submodule)
+        if snapshot is None:
+            snapshot = getattr(submodule, '_run_torch_master_expert_bias_fp32', None)
+        if snapshot is None:
+            continue
+        expert_bias.data = snapshot.to(device=expert_bias.device, dtype=snapshot.dtype)
+        restored += 1
+    return restored
+
+
 def param_is_not_shared(param):  # pylint: disable=missing-function-docstring
     return not hasattr(param, 'shared') or not param.shared
 
@@ -434,6 +468,10 @@ def __init__(self, config: TransformerConfig, module: torch.nn.Module):
         self.vp_stage = getattr(module, 'vp_stage', None)
         self.pg_collection = getattr(module, 'pg_collection', None)
 
+        expert_bias_snapshots = {}
+        if self.fp16 or self.bf16:
+            expert_bias_snapshots = _snapshot_expert_bias_fp32(module)
+
         if self.fp16:
             self.add_module('module', module.half())
 
@@ -449,6 +487,12 @@ def float16_convertor(val):
         else:
             raise Exception('Either config.fp16 or config.bf16 should be True.')
 
+        if expert_bias_snapshots:
+            _restore_expert_bias_fp32(self.module, expert_bias_snapshots)
+
+        if getattr(config, 'moe_router_bias_update_rate', 0.0) != 0.0:
+            config.moe_router_bias_update_rate = 0.0
+
         self.float16_convertor = float16_convertor
 
     def set_input_tensor(self, input_tensor):  # pylint: disable=missing-function-docstring

From 6a093d819256dea47702c1549e01baac8f6fc6f2 Mon Sep 17 00:00:00 2001
From: zhengshengning <ningzhengsheng@baidu.com>
Date: Wed, 24 Jun 2026 17:24:05 +0800
Subject: [PATCH 3/7] Router TE GEMM

---
 megatron/core/transformer/moe/moe_utils.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py
index 4c424c74b0b..123cd96a580 100644
--- a/megatron/core/transformer/moe/moe_utils.py
+++ b/megatron/core/transformer/moe/moe_utils.py
@@ -1322,7 +1322,11 @@ def forward(
         inp_shape = inp.shape
         inp = inp.view(-1, inp_shape[-1])
 
-        if te_general_gemm is not None and router_dtype != torch.float64:
+        # 禁用 router TE GEMM，强制走 torch.mm 以对齐 PaddleFleet。
+        # te_general_gemm 在 fp32 router_dtype 下与 PF 的 matmul 选不同 cuBLAS 算法，
+        # 导致 gate_logits 末位 diff，叠加 topk 离散选择后 router_output_0/1 翻转。
+        # if te_general_gemm is not None and router_dtype != torch.float64:
+        if False:
             output = te_general_gemm(weight, inp, router_dtype, layout="TN", bias=bias)
             output = output[0]
         elif bias is None:
@@ -1355,7 +1359,9 @@ def backward(
         inp = inp.view(-1, inp_shape[-1])
         grad_output = grad_output.view(-1, grad_shape[-1])
 
-        if te_general_gemm is not None and ctx.router_dtype != torch.float64:
+        # 禁用 TE GEMM 以对齐 PF 精度
+        # if te_general_gemm is not None and ctx.router_dtype != torch.float64:
+        if False:
             grad_input = te_general_gemm(
                 weight.to(ctx.router_dtype), grad_output, ctx.router_dtype, layout="NN", grad=True
             )

From 4578ae6292c84a6a4502fb31f44d9a522b936110 Mon Sep 17 00:00:00 2001
From: zhengshengning <ningzhengsheng@baidu.com>
Date: Fri, 26 Jun 2026 14:31:59 +0800
Subject: [PATCH 4/7] vocab_parallel_cross_entropy

---
 .../common/language_module/language_module.py      | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/megatron/core/models/common/language_module/language_module.py b/megatron/core/models/common/language_module/language_module.py
index 3c6b7c4ab8d..9b5fb3a91d1 100644
--- a/megatron/core/models/common/language_module/language_module.py
+++ b/megatron/core/models/common/language_module/language_module.py
@@ -4,6 +4,7 @@
 from typing import Optional, Tuple
 
 import torch
+import torch.nn.functional as F
 from torch import Tensor
 
 from megatron.core import parallel_state, tensor_parallel
@@ -138,7 +139,18 @@ def compute_language_model_loss(self, labels: Tensor, logits: Tensor) -> Tensor:
         """
         # [b s] => [s b]
         labels = labels.transpose(0, 1).contiguous()
-        if self.config.cross_entropy_loss_fusion:
+        # 【修复的问题描述】：Megatron 默认走 `tensor_parallel.vocab_parallel_cross_entropy`，
+        # 在 TP=1 场景下其内部累加顺序与 paddle `nn.CrossEntropyLoss` / `F.cross_entropy`
+        # 不一致，导致两框架 LM loss 末位存在 diff。此处在 TP=1 且 logits 未做 vocab 切分
+        # 时切换到 `torch.nn.functional.cross_entropy`，与 Paddle 侧 CE 路径对齐。
+        if parallel_state.get_tensor_model_parallel_world_size() == 1 and logits.ndim == 3:
+            loss = F.cross_entropy(
+                logits.float().reshape(-1, logits.shape[-1]),
+                labels.reshape(-1),
+                reduction="none",
+                ignore_index=-100,
+            ).view_as(labels)
+        elif self.config.cross_entropy_loss_fusion:
             if self.config.cross_entropy_fusion_impl == 'te':
                 if te_parallel_cross_entropy is not None:
                     labels = torch.as_strided(labels, labels.size(), (labels.size()[1], 1))

From 55d5ad5e8e9c220f4e001afaca97b66930b2e671 Mon Sep 17 00:00:00 2001
From: zhengshengning <ningzhengsheng@baidu.com>
Date: Fri, 26 Jun 2026 14:43:10 +0800
Subject: [PATCH 5/7] scatter_add_

---
 megatron/core/transformer/moe/moe_utils.py | 35 ++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py
index 123cd96a580..0b3d94a9648 100644
--- a/megatron/core/transformer/moe/moe_utils.py
+++ b/megatron/core/transformer/moe/moe_utils.py
@@ -2,6 +2,7 @@
 
 import functools
 import math
+import os
 from dataclasses import dataclass
 from typing import List, Optional, Tuple, Union
 
@@ -52,6 +53,29 @@
     ) = (None, None, None, None, None, None, None, None, None, None)
 
 
+
+def _fp32_accum_unpermute(permuted_tokens: torch.Tensor, sorted_indices: torch.Tensor, restore_shape):
+    # 【修复的问题描述】：MoE unpermute 阶段 `scatter_add_` 在 bf16 下走 atomic 累加，
+    # 多 expert 输出回写到同一 token 行时累加顺序不可复现，与 PaddleFleet 末位 diff。
+    # 把 permuted_tokens 提升到 fp32 后用 scatter_add_ 累加，再 cast 回原 dtype，
+    # 单次 round，复刻 PF 侧确定性 fp32 累加路径。
+    if len(restore_shape) != 2:
+        return None
+
+    hidden = int(restore_shape[-1])
+    output_tokens = torch.zeros(
+        restore_shape,
+        dtype=torch.float32,
+        device=permuted_tokens.device,
+    )
+    output_tokens.scatter_add_(
+        0,
+        sorted_indices.unsqueeze(1).expand(-1, hidden),
+        permuted_tokens.to(torch.float32),
+    )
+    return output_tokens.to(dtype=permuted_tokens.dtype)
+
+
 # MOE logging
 _MOE_LAYER_WISE_LOGGING_TRACKER: dict = {}
 
@@ -483,6 +507,17 @@ def unpermute(
     _, hidden = restore_shape
     input_dtype = permuted_tokens.dtype
 
+    # 【修复的问题描述】：MoE unpermute 阶段 `scatter_add_` 在 bf16 下走 atomic 累加，
+    # 多 expert 输出回写到同一 token 行时累加顺序不可复现，与 PaddleFleet 末位 diff。
+    # 在 unpermute（无 probs、非 drop_and_pad）的标准路径上改走 fp32 累加。
+    if (
+        probs is None
+        and not drop_and_pad
+    ):
+        fp32_output = _fp32_accum_unpermute(permuted_tokens, sorted_indices, restore_shape)
+        if fp32_output is not None:
+            return fp32_output
+
     if probs is not None:
         assert routing_map is not None, "Mask must be provided to permute the probs."
         if drop_and_pad:

From 63f5f2333cd3027becd240fb1b06d8016fc1bec9 Mon Sep 17 00:00:00 2001
From: zhengshengning <ningzhengsheng@baidu.com>
Date: Fri, 26 Jun 2026 15:01:56 +0800
Subject: [PATCH 6/7] SwiGLU

---
 megatron/core/transformer/mlp.py | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/megatron/core/transformer/mlp.py b/megatron/core/transformer/mlp.py
index 8a19fef87ec..1d3686b4a8d 100644
--- a/megatron/core/transformer/mlp.py
+++ b/megatron/core/transformer/mlp.py
@@ -307,9 +307,18 @@ def forward(
         else:
             if bias_parallel is not None:
                 intermediate_parallel = intermediate_parallel + bias_parallel
+            # 【修复的问题描述】：MoE expert 内 SwiGLU 与 router prob 相乘的计算精度对齐。
+            # PaddleFleet 的 `fused_swiglu_scale` CUDA kernel 在 fp32 下完成 SwiGLU
+            # 激活并乘上 per_token_scale（router prob），最后一次性 round 回 bf16；
+            # 而 Megatron 原实现是 bf16 SwiGLU + bf16 乘 prob，存在两次 bf16 round，
+            # 与 PF 末位有 diff。这里在存在 per_token_scale 时把 GLU 提升到 fp32 计算，
+            # 配合下方 fp32 乘 prob 后再 cast 回 bf16，对齐 PF 单次 round 的语义。
+            _glu_fp32 = per_token_scale is not None
             if self.config.gated_linear_unit:
 
                 def glu(x):
+                    if _glu_fp32:
+                        x = x.to(torch.float32)
                     x_glu, x_linear = torch.chunk(x, 2, dim=-1)
                     if (val := self.config.activation_func_clamp_value) is not None:
                         x_glu = x_glu.clamp(min=None, max=val)
@@ -320,12 +329,27 @@ def glu(x):
 
                 intermediate_parallel = glu(intermediate_parallel)
             else:
-                intermediate_parallel = self.activation_func(intermediate_parallel)
+                # 【修复的问题描述】：MoE expert 内 SwiGLU 与 router prob 相乘的计算精度对齐。
+                # 非 GLU 分支同样在存在 per_token_scale 时把激活提升到 fp32 计算，
+                # 保持与 PaddleFleet fused_swiglu_scale 一致的单次 round 路径。
+                if _glu_fp32:
+                    intermediate_parallel = self.activation_func(
+                        intermediate_parallel.to(torch.float32)
+                    )
+                else:
+                    intermediate_parallel = self.activation_func(intermediate_parallel)
 
             if per_token_scale is not None:
-                original_dtype = intermediate_parallel.dtype
-                intermediate_parallel = intermediate_parallel * per_token_scale.unsqueeze(-1)
+                # 【修复的问题描述】：MoE expert 内 SwiGLU 与 router prob 相乘的计算精度对齐。
+                # GLU 已在 fp32 下计算，这里把 per_token_scale 也 cast 到 fp32 相乘，
+                # 最后一次性 cast 回原始 bf16 dtype，对齐 PaddleFleet fused_swiglu_scale
+                # 「fp32 激活 × fp32 prob → 单次 bf16 round」的数值路径。
+                original_dtype = hidden_states.dtype
+                intermediate_parallel = intermediate_parallel * per_token_scale.unsqueeze(-1).to(
+                    intermediate_parallel.dtype
+                )
                 intermediate_parallel = intermediate_parallel.to(original_dtype)
+
         nvtx_range_pop(suffix="activation")
 
         # [s, b, h]

From 6a4c0cbbbe3f2f01a6c8efd075641527ac57b25c Mon Sep 17 00:00:00 2001
From: zhengshengning <ningzhengsheng@baidu.com>
Date: Tue, 7 Jul 2026 20:58:34 +0800
Subject: [PATCH 7/7] add use_accuracy_compatible

---
 .../common/language_module/language_module.py | 16 +++++++++-
 megatron/core/transformer/mlp.py              | 29 ++++++++++++-------
 megatron/core/transformer/module.py           | 16 ++++++++--
 megatron/core/transformer/moe/experts.py      | 19 ++++++++++--
 megatron/core/transformer/moe/moe_utils.py    | 29 +++++++++++++++----
 5 files changed, 87 insertions(+), 22 deletions(-)

diff --git a/megatron/core/models/common/language_module/language_module.py b/megatron/core/models/common/language_module/language_module.py
index 9b5fb3a91d1..7984f27c882 100644
--- a/megatron/core/models/common/language_module/language_module.py
+++ b/megatron/core/models/common/language_module/language_module.py
@@ -34,6 +34,15 @@
 )
 
 
+def _use_accuracy_compatible() -> bool:
+    """Runtime switch for the PaddleFleet<->Megatron bit-alignment patches.
+
+    Driven by ms-swift's ``use_accuracy_compatible`` arg via the ``USE_ACCURACY_COMPATIBLE``
+    env var. Defaults to False so that unpatched usage keeps the original Megatron logic.
+    """
+    return os.environ.get('USE_ACCURACY_COMPATIBLE', '0') == '1'
+
+
 class LanguageModule(MegatronModule):
     """Base language module that has common helper functions used across GPT, BERT etc.
 
@@ -143,7 +152,12 @@ def compute_language_model_loss(self, labels: Tensor, logits: Tensor) -> Tensor:
         # 在 TP=1 场景下其内部累加顺序与 paddle `nn.CrossEntropyLoss` / `F.cross_entropy`
         # 不一致，导致两框架 LM loss 末位存在 diff。此处在 TP=1 且 logits 未做 vocab 切分
         # 时切换到 `torch.nn.functional.cross_entropy`，与 Paddle 侧 CE 路径对齐。
-        if parallel_state.get_tensor_model_parallel_world_size() == 1 and logits.ndim == 3:
+        # 由 use_accuracy_compatible 控制：关闭时走原始 vocab_parallel_cross_entropy / fusion 路径。
+        if (
+            _use_accuracy_compatible()
+            and parallel_state.get_tensor_model_parallel_world_size() == 1
+            and logits.ndim == 3
+        ):
             loss = F.cross_entropy(
                 logits.float().reshape(-1, logits.shape[-1]),
                 labels.reshape(-1),
diff --git a/megatron/core/transformer/mlp.py b/megatron/core/transformer/mlp.py
index 1d3686b4a8d..60d28e0e498 100644
--- a/megatron/core/transformer/mlp.py
+++ b/megatron/core/transformer/mlp.py
@@ -25,7 +25,7 @@
 )
 from megatron.core.fusions.fused_bias_gelu import bias_gelu_impl
 from megatron.core.fusions.fused_bias_swiglu import bias_swiglu_impl, weighted_bias_swiglu_impl
-from megatron.core.transformer.module import MegatronModule
+from megatron.core.transformer.module import MegatronModule, _use_accuracy_compatible
 from megatron.core.transformer.transformer_config import TransformerConfig
 from megatron.core.typed_torch import apply_module, not_none
 from megatron.core.utils import (
@@ -313,7 +313,8 @@ def forward(
             # 而 Megatron 原实现是 bf16 SwiGLU + bf16 乘 prob，存在两次 bf16 round，
             # 与 PF 末位有 diff。这里在存在 per_token_scale 时把 GLU 提升到 fp32 计算，
             # 配合下方 fp32 乘 prob 后再 cast 回 bf16，对齐 PF 单次 round 的语义。
-            _glu_fp32 = per_token_scale is not None
+            # 由 use_accuracy_compatible 控制：关闭时保留原始 bf16 计算路径。
+            _glu_fp32 = per_token_scale is not None and _use_accuracy_compatible()
             if self.config.gated_linear_unit:
 
                 def glu(x):
@@ -340,15 +341,21 @@ def glu(x):
                     intermediate_parallel = self.activation_func(intermediate_parallel)
 
             if per_token_scale is not None:
-                # 【修复的问题描述】：MoE expert 内 SwiGLU 与 router prob 相乘的计算精度对齐。
-                # GLU 已在 fp32 下计算，这里把 per_token_scale 也 cast 到 fp32 相乘，
-                # 最后一次性 cast 回原始 bf16 dtype，对齐 PaddleFleet fused_swiglu_scale
-                # 「fp32 激活 × fp32 prob → 单次 bf16 round」的数值路径。
-                original_dtype = hidden_states.dtype
-                intermediate_parallel = intermediate_parallel * per_token_scale.unsqueeze(-1).to(
-                    intermediate_parallel.dtype
-                )
-                intermediate_parallel = intermediate_parallel.to(original_dtype)
+                if _glu_fp32:
+                    # 【修复的问题描述】：MoE expert 内 SwiGLU 与 router prob 相乘的计算精度对齐。
+                    # GLU 已在 fp32 下计算，这里把 per_token_scale 也 cast 到 fp32 相乘，
+                    # 最后一次性 cast 回原始 bf16 dtype，对齐 PaddleFleet fused_swiglu_scale
+                    # 「fp32 激活 × fp32 prob → 单次 bf16 round」的数值路径。
+                    original_dtype = hidden_states.dtype
+                    intermediate_parallel = intermediate_parallel * per_token_scale.unsqueeze(-1).to(
+                        intermediate_parallel.dtype
+                    )
+                    intermediate_parallel = intermediate_parallel.to(original_dtype)
+                else:
+                    # 原始路径：bf16 SwiGLU 输出直接乘 bf16 prob。
+                    original_dtype = intermediate_parallel.dtype
+                    intermediate_parallel = intermediate_parallel * per_token_scale.unsqueeze(-1)
+                    intermediate_parallel = intermediate_parallel.to(original_dtype)
 
         nvtx_range_pop(suffix="activation")
 
diff --git a/megatron/core/transformer/module.py b/megatron/core/transformer/module.py
index 41c924fb8a7..afac302d7b4 100644
--- a/megatron/core/transformer/module.py
+++ b/megatron/core/transformer/module.py
@@ -57,6 +57,17 @@ def _restore_expert_bias_fp32(module, snapshots):
     return restored
 
 
+def _use_accuracy_compatible() -> bool:
+    """Runtime switch for the PaddleFleet<->Megatron bit-alignment patches.
+
+    Driven by ms-swift's ``use_accuracy_compatible`` arg via the ``USE_ACCURACY_COMPATIBLE``
+    env var. Defaults to False so that unpatched usage keeps the original Megatron logic.
+    """
+    import os
+
+    return os.environ.get('USE_ACCURACY_COMPATIBLE', '0') == '1'
+
+
 def param_is_not_shared(param):  # pylint: disable=missing-function-docstring
     return not hasattr(param, 'shared') or not param.shared
 
@@ -469,7 +480,8 @@ def __init__(self, config: TransformerConfig, module: torch.nn.Module):
         self.pg_collection = getattr(module, 'pg_collection', None)
 
         expert_bias_snapshots = {}
-        if self.fp16 or self.bf16:
+        _accuracy_compatible = _use_accuracy_compatible()
+        if _accuracy_compatible and (self.fp16 or self.bf16):
             expert_bias_snapshots = _snapshot_expert_bias_fp32(module)
 
         if self.fp16:
@@ -490,7 +502,7 @@ def float16_convertor(val):
         if expert_bias_snapshots:
             _restore_expert_bias_fp32(self.module, expert_bias_snapshots)
 
-        if getattr(config, 'moe_router_bias_update_rate', 0.0) != 0.0:
+        if _accuracy_compatible and getattr(config, 'moe_router_bias_update_rate', 0.0) != 0.0:
             config.moe_router_bias_update_rate = 0.0
 
         self.float16_convertor = float16_convertor
diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py
index ddff045033b..b8e83e2ef19 100644
--- a/megatron/core/transformer/moe/experts.py
+++ b/megatron/core/transformer/moe/experts.py
@@ -724,6 +724,17 @@ def forward(
             return super().forward(permuted_local_hidden_states, tokens_per_expert, permuted_probs)
 
 
+def _use_accuracy_compatible() -> bool:
+    """Runtime switch for the PaddleFleet<->Megatron bit-alignment patches.
+
+    Driven by ms-swift's ``use_accuracy_compatible`` arg via the ``USE_ACCURACY_COMPATIBLE``
+    env var. Defaults to False so that unpatched usage keeps the original Megatron logic.
+    """
+    import os
+
+    return os.environ.get('USE_ACCURACY_COMPATIBLE', '0') == '1'
+
+
 class _SeqMLPProxy:
     """Expose GroupedMLP-compatible ``weight{i}`` / ``bias{i}`` access on top of
     ``SequentialMLP.local_experts[i].linear_fc{1,2}``.
@@ -731,7 +742,6 @@ class _SeqMLPProxy:
     Required by upper layers (e.g. ms-swift's ``GPTBridge._set_mlp_state``) that
     probe ``mg_mlp.linear_fc1`` / ``linear_fc2`` like GroupedMLP.
     """
-
     def __init__(self, experts, attr):
         self._experts = experts
         self._attr = attr
@@ -792,8 +802,11 @@ def __init__(
         # Upper layers (e.g. ms-swift gpt_bridge._set_mlp_state) probe
         # `mg_mlp.linear_fc1` / `linear_fc2` and expect `weight{i}`/`bias{i}` access
         # like GroupedMLP. Inject a thin proxy that forwards to local_experts[i].
-        self.linear_fc1 = _SeqMLPProxy(self.local_experts, 'linear_fc1')
-        self.linear_fc2 = _SeqMLPProxy(self.local_experts, 'linear_fc2')
+        # Gated by use_accuracy_compatible; otherwise SequentialMLP keeps its original
+        # interface (no linear_fc1/linear_fc2 attributes).
+        if _use_accuracy_compatible():
+            self.linear_fc1 = _SeqMLPProxy(self.local_experts, 'linear_fc1')
+            self.linear_fc2 = _SeqMLPProxy(self.local_experts, 'linear_fc2')
 
     def _pad_tensor_for_quantization(self, hidden, probs):
         """Padding tensor shape to multiples of 16/32."""
diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py
index 0b3d94a9648..205e308c3c5 100644
--- a/megatron/core/transformer/moe/moe_utils.py
+++ b/megatron/core/transformer/moe/moe_utils.py
@@ -54,6 +54,15 @@
 
 
 
+def _use_accuracy_compatible() -> bool:
+    """Runtime switch for the PaddleFleet<->Megatron bit-alignment patches.
+
+    Driven by ms-swift's ``use_accuracy_compatible`` arg via the ``USE_ACCURACY_COMPATIBLE``
+    env var. Defaults to False so that unpatched usage keeps the original Megatron logic.
+    """
+    return os.environ.get('USE_ACCURACY_COMPATIBLE', '0') == '1'
+
+
 def _fp32_accum_unpermute(permuted_tokens: torch.Tensor, sorted_indices: torch.Tensor, restore_shape):
     # 【修复的问题描述】：MoE unpermute 阶段 `scatter_add_` 在 bf16 下走 atomic 累加，
     # 多 expert 输出回写到同一 token 行时累加顺序不可复现，与 PaddleFleet 末位 diff。
@@ -510,8 +519,10 @@ def unpermute(
     # 【修复的问题描述】：MoE unpermute 阶段 `scatter_add_` 在 bf16 下走 atomic 累加，
     # 多 expert 输出回写到同一 token 行时累加顺序不可复现，与 PaddleFleet 末位 diff。
     # 在 unpermute（无 probs、非 drop_and_pad）的标准路径上改走 fp32 累加。
+    # 由 use_accuracy_compatible 控制，关闭时保留原始 scatter_add_ 路径。
     if (
-        probs is None
+        _use_accuracy_compatible()
+        and probs is None
         and not drop_and_pad
     ):
         fp32_output = _fp32_accum_unpermute(permuted_tokens, sorted_indices, restore_shape)
@@ -1360,8 +1371,12 @@ def forward(
         # 禁用 router TE GEMM，强制走 torch.mm 以对齐 PaddleFleet。
         # te_general_gemm 在 fp32 router_dtype 下与 PF 的 matmul 选不同 cuBLAS 算法，
         # 导致 gate_logits 末位 diff，叠加 topk 离散选择后 router_output_0/1 翻转。
-        # if te_general_gemm is not None and router_dtype != torch.float64:
-        if False:
+        # 由 use_accuracy_compatible 控制：关闭时保留原始 TE GEMM 路径。
+        if (
+            not _use_accuracy_compatible()
+            and te_general_gemm is not None
+            and router_dtype != torch.float64
+        ):
             output = te_general_gemm(weight, inp, router_dtype, layout="TN", bias=bias)
             output = output[0]
         elif bias is None:
@@ -1395,8 +1410,12 @@ def backward(
         grad_output = grad_output.view(-1, grad_shape[-1])
 
         # 禁用 TE GEMM 以对齐 PF 精度
-        # if te_general_gemm is not None and ctx.router_dtype != torch.float64:
-        if False:
+        # 由 use_accuracy_compatible 控制：关闭时保留原始 TE GEMM 路径。
+        if (
+            not _use_accuracy_compatible()
+            and te_general_gemm is not None
+            and ctx.router_dtype != torch.float64
+        ):
             grad_input = te_general_gemm(
                 weight.to(ctx.router_dtype), grad_output, ctx.router_dtype, layout="NN", grad=True
             )