[CUDA] Deprecate SkipLayerNorm strict mode (#29388)

tianleiwu · web-flow · commit 7e8fc2958d92 · 2026-06-29T20:38:36.000-07:00
### Description Removes the deprecated `enable_skip_layer_norm_strict_mode` CUDA provider option and its associated "strict mode" code path from the CUDA `SkipLayerNorm` kernel. Strict mode previously routed `SkipLayerNorm` through the `LayerNormalization` kernel to gain fp32-accumulation accuracy at the cost of performance. After #28682 made `SkipLayerNorm`/`EmbedLayerNorm` CUDA kernels always accumulate in fp32, the strict-mode path is redundant: the default kernel already provides the same accuracy with better performance. This change deletes the now-dead branch and the plumbing that fed it. ### Key Changes - **`skip_layer_norm.cc`**: Remove the `strict_` branch that called `HostApplyLayerNorm`; always launch `LaunchSkipLayerNormKernel`. Drop the now-unused `layer_norm_impl.h` include and the input/skip same-shape strict-mode shape check. - **`skip_layer_norm.h`**: Remove the `strict_` member. - **`cuda_execution_provider.h`**: Remove `IsSkipLayerNormInStrictMode()`. - **`cuda_kernel_adapter.h`**: Remove `GetCudaKernelAdapterSkipLayerNormStrictMode()` shim. - **`cuda_provider_options.h`**: Keep `enable_skip_layer_norm_strict_mode` field for ABI/back-compat but mark it deprecated and ignored. - **`skiplayernorm_op_test.cc`**: Drop the redundant strict-mode test passes; tests now run a single default path. The provider option is retained (ignored) to preserve backward compatibility — existing configs that set it continue to work without error. ### Motivation Follow-up cleanup to #28682, which switched the CUDA kernels to fp32 accumulation, making strict mode obsolete. ### Testing - `skiplayernorm_op_test.cc` covers fp16/fp32/bf16 default path; strict-mode passes removed.
diff --git a/include/onnxruntime/core/providers/cuda/cuda_provider_options.h b/include/onnxruntime/core/providers/cuda/cuda_provider_options.h
@@ -33,8 +33,8 @@ struct OrtCUDAProviderOptionsV2 {
   int tunable_op_enable = 0;                                                                                   // flag specifying if TunableOp is enabled.
   int tunable_op_tuning_enable = 0;                                                                            // flag specifying if TunableOp is enabled for tuning, this relies on TunableOp is enabled.
   int tunable_op_max_tuning_duration_ms = 0;                                                                   // Max tuning duration time limit for TunableOp.
-  int enable_skip_layer_norm_strict_mode = 0;                                                                  // flag specifying if SkipLayerNorm is in strict mode. If true, use LayerNormalization kernel.
-                                                                                                               // The strict mode has better accuracy but lower performance.
+  int enable_skip_layer_norm_strict_mode = 0;                                                                  // [Deprecated] Accepted for ABI/back-compat but not stored in EP info. SkipLayerNorm always accumulates in fp32.
+                                                                                                               // Setting it has no effect on computation or output.
   int prefer_nhwc = 0;                                                                                         // make the CUDA EP NHWC preferred
   int use_ep_level_unified_stream = 0;                                                                         // flag specifying if ep level stream is used or not
   int use_tf32 = 1;                                                                                            // use TF32
diff --git a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.cc b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.cc
@@ -2,7 +2,6 @@
 // Licensed under the MIT License.
 
 #include "core/providers/cuda/cuda_common.h"
-#include "core/providers/cuda/nn/layer_norm_impl.h"
 #include "core/common/narrow.h"
 #include "skip_layer_norm.h"
 #include "skip_layer_norm_impl.h"
@@ -42,26 +41,14 @@ template <typename T, bool Simplified>
 SkipLayerNorm<T, Simplified>::SkipLayerNorm(const OpKernelInfo& op_kernel_info) : CudaKernel(op_kernel_info) {
   ORT_ENFORCE(op_kernel_info.GetAttr<float>("epsilon", &epsilon_).IsOK());
   ORT_ENFORCE(epsilon_ >= 0);
-
-#ifdef BUILD_CUDA_EP_AS_PLUGIN
-  // Plugin adapter cannot static_cast to CUDAExecutionProvider directly.
-  // Use the adapter shim that reads the config from the per-EP runtime map.
-  strict_ = onnxruntime::cuda::GetCudaKernelAdapterSkipLayerNormStrictMode(op_kernel_info.GetExecutionProvider());
-#else
-  const CUDAExecutionProvider* cuda_ep = static_cast<const CUDAExecutionProvider*>(op_kernel_info.GetExecutionProvider());
-  strict_ = cuda_ep->IsSkipLayerNormInStrictMode();
-#endif
+  // Note: the enable_skip_layer_norm_strict_mode provider option is deprecated and ignored.
+  // The kernel always accumulates in fp32, so the previous strict-mode path is no longer needed.
 }
 
 template <typename T, bool Simplified>
 Status SkipLayerNorm<T, Simplified>::ComputeInternal(OpKernelContext* ctx) const {
   const Tensor* input = ctx->Input<Tensor>(0);
   const Tensor* skip = ctx->Input<Tensor>(1);
-  if (strict_ && skip->Shape() != input->Shape()) {
-    return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
-                           "'input' and 'skip' shall have same shape when enable_skip_layer_norm_strict_mode is True");
-  }
-
   const Tensor* gamma = ctx->Input<Tensor>(2);
 
   const Tensor* beta = Simplified ? nullptr : ctx->Input<Tensor>(3);
@@ -94,53 +81,34 @@ Status SkipLayerNorm<T, Simplified>::ComputeInternal(OpKernelContext* ctx) const
 
   const int skip_size = onnxruntime::narrow<int>(skip->Shape().Size());
 
-  if (strict_) {
-    HostApplyLayerNorm<CudaT, float, CudaT, Simplified>(
-        GetDeviceProp(),
+  if constexpr (std::is_same_v<T, BFloat16>) {
+    LaunchSkipLayerNormKernel<nv_bfloat16, Simplified>(
         Stream(ctx),
-        reinterpret_cast<CudaT*>(output->MutableData<T>()),                             // Y_data
-        nullptr,                                                                        // mean_data
-        nullptr,                                                                        // inv_var_data
-        reinterpret_cast<const CudaT*>(input->Data<T>()),                               // X_data
-        row_count,                                                                      // n1
-        hidden_size,                                                                    // n2
-        (double)epsilon_,                                                               // epsilon
-        reinterpret_cast<const CudaT*>(gamma->Data<T>()),                               // gamma
-        (beta != nullptr) ? reinterpret_cast<const CudaT*>(beta->Data<T>()) : nullptr,  // beta
-        0,                                                                              // no broadcast for gamma/beta
-        reinterpret_cast<const CudaT*>(skip->Data<T>()),                                // skip or residual to add
-        (bias != nullptr) ? reinterpret_cast<const CudaT*>(bias->Data<T>()) : nullptr,  // bias to add
-        sum_output != nullptr ? reinterpret_cast<CudaT*>(sum_output->MutableData<T>()) : nullptr);
+        reinterpret_cast<nv_bfloat16*>(output->MutableData<T>()),
+        sum_output != nullptr ? reinterpret_cast<nv_bfloat16*>(sum_output->MutableData<T>()) : nullptr,
+        reinterpret_cast<const nv_bfloat16*>(input->Data<T>()),
+        reinterpret_cast<const nv_bfloat16*>(skip->Data<T>()),
+        (bias != nullptr) ? reinterpret_cast<const nv_bfloat16*>(bias->Data<T>()) : nullptr,
+        reinterpret_cast<const nv_bfloat16*>(gamma->Data<T>()),
+        (beta != nullptr) ? reinterpret_cast<const nv_bfloat16*>(beta->Data<T>()) : nullptr,
+        epsilon_,
+        hidden_size,
+        row_count,
+        skip_size);
   } else {
-    if constexpr (std::is_same_v<T, BFloat16>) {
-      LaunchSkipLayerNormKernel<nv_bfloat16, Simplified>(
-          Stream(ctx),
-          reinterpret_cast<nv_bfloat16*>(output->MutableData<T>()),
-          sum_output != nullptr ? reinterpret_cast<nv_bfloat16*>(sum_output->MutableData<T>()) : nullptr,
-          reinterpret_cast<const nv_bfloat16*>(input->Data<T>()),
-          reinterpret_cast<const nv_bfloat16*>(skip->Data<T>()),
-          (bias != nullptr) ? reinterpret_cast<const nv_bfloat16*>(bias->Data<T>()) : nullptr,
-          reinterpret_cast<const nv_bfloat16*>(gamma->Data<T>()),
-          (beta != nullptr) ? reinterpret_cast<const nv_bfloat16*>(beta->Data<T>()) : nullptr,
-          epsilon_,
-          hidden_size,
-          row_count,
-          skip_size);
-    } else {
-      LaunchSkipLayerNormKernel<CudaT, Simplified>(
-          Stream(ctx),
-          reinterpret_cast<CudaT*>(output->MutableData<T>()),
-          sum_output != nullptr ? reinterpret_cast<CudaT*>(sum_output->MutableData<T>()) : nullptr,
-          reinterpret_cast<const CudaT*>(input->Data<T>()),
-          reinterpret_cast<const CudaT*>(skip->Data<T>()),
-          (bias != nullptr) ? reinterpret_cast<const CudaT*>(bias->Data<T>()) : nullptr,
-          reinterpret_cast<const CudaT*>(gamma->Data<T>()),
-          (beta != nullptr) ? reinterpret_cast<const CudaT*>(beta->Data<T>()) : nullptr,
-          epsilon_,
-          hidden_size,
-          row_count,
-          skip_size);
-    }
+    LaunchSkipLayerNormKernel<CudaT, Simplified>(
+        Stream(ctx),
+        reinterpret_cast<CudaT*>(output->MutableData<T>()),
+        sum_output != nullptr ? reinterpret_cast<CudaT*>(sum_output->MutableData<T>()) : nullptr,
+        reinterpret_cast<const CudaT*>(input->Data<T>()),
+        reinterpret_cast<const CudaT*>(skip->Data<T>()),
+        (bias != nullptr) ? reinterpret_cast<const CudaT*>(bias->Data<T>()) : nullptr,
+        reinterpret_cast<const CudaT*>(gamma->Data<T>()),
+        (beta != nullptr) ? reinterpret_cast<const CudaT*>(beta->Data<T>()) : nullptr,
+        epsilon_,
+        hidden_size,
+        row_count,
+        skip_size);
   }
 
   CUDA_RETURN_IF_ERROR(cudaGetLastError());
diff --git a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.h b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.h
@@ -19,7 +19,6 @@ class SkipLayerNorm final : public CudaKernel {
 
  private:
   float epsilon_;
-  bool strict_;
 };
 
 }  // namespace cuda
diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.h b/onnxruntime/core/providers/cuda/cuda_execution_provider.h
@@ -83,7 +83,6 @@ class CUDAExecutionProvider : public IExecutionProvider {
   bool DoCopyOnDefaultStream() const { return info_.do_copy_in_default_stream; }
   bool GetCudnnConvUseMaxWorkspace() const { return info_.cudnn_conv_use_max_workspace; }
   bool GetCudnnConv1dPadToNc1d() const { return info_.cudnn_conv1d_pad_to_nc1d; }
-  bool IsSkipLayerNormInStrictMode() const { return info_.enable_skip_layer_norm_strict_mode; }
   bool IsNHWCPreferred() const { return info_.prefer_nhwc; }
   bool IsFuseConvBias() const { return info_.fuse_conv_bias; }
   bool UseTF32() const { return info_.use_tf32; }
diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc
@@ -30,6 +30,7 @@ constexpr const char* kCudnnConv1dPadToNc1d = "cudnn_conv1d_pad_to_nc1d";
 constexpr const char* kTunableOpEnable = "tunable_op_enable";
 constexpr const char* kTunableOpTuningEnable = "tunable_op_tuning_enable";
 constexpr const char* kTunableOpMaxTuningDurationMs = "tunable_op_max_tuning_duration_ms";
+// [Deprecated] Accepted but ignored: SkipLayerNorm always accumulates in fp32.
 constexpr const char* kEnableSkipLayerNormStrictMode = "enable_skip_layer_norm_strict_mode";
 constexpr const char* kPreferNHWCMode = "prefer_nhwc";
 constexpr const char* kUseEPLevelUnifiedStream = "use_ep_level_unified_stream";
@@ -115,7 +116,15 @@ CUDAExecutionProviderInfo CUDAExecutionProviderInfo::FromProviderOptions(const P
           .AddAssignmentToReference(cuda::provider_option_names::kCudnnConvUseMaxWorkspace, info.cudnn_conv_use_max_workspace)
           .AddAssignmentToReference(cuda::provider_option_names::kEnableCudaGraph, info.enable_cuda_graph)
           .AddAssignmentToReference(cuda::provider_option_names::kCudnnConv1dPadToNc1d, info.cudnn_conv1d_pad_to_nc1d)
-          .AddAssignmentToReference(cuda::provider_option_names::kEnableSkipLayerNormStrictMode, info.enable_skip_layer_norm_strict_mode)
+          .AddValueParser(
+              cuda::provider_option_names::kEnableSkipLayerNormStrictMode,
+              [](const std::string& value_str) -> Status {
+                // [Deprecated] Accept the option for backward compatibility, but do not store it:
+                // SkipLayerNorm always accumulates in fp32, so strict mode has no effect.
+                bool ignored = false;
+                ORT_RETURN_IF_ERROR(ParseStringWithClassicLocale(value_str, ignored));
+                return Status::OK();
+              })
           .AddAssignmentToReference(cuda::provider_option_names::kPreferNHWCMode, info.prefer_nhwc)
           .AddAssignmentToReference(cuda::provider_option_names::kUseEPLevelUnifiedStream, info.use_ep_level_unified_stream)
           .AddAssignmentToReference(cuda::provider_option_names::kUseTF32, info.use_tf32)
@@ -170,7 +179,6 @@ ProviderOptions CUDAExecutionProviderInfo::ToProviderOptions(const CUDAExecution
       {cuda::provider_option_names::kTunableOpEnable, MakeStringWithClassicLocale(info.tunable_op.enable)},
       {cuda::provider_option_names::kTunableOpTuningEnable, MakeStringWithClassicLocale(info.tunable_op.tuning_enable)},
       {cuda::provider_option_names::kTunableOpMaxTuningDurationMs, MakeStringWithClassicLocale(info.tunable_op.max_tuning_duration_ms)},
-      {cuda::provider_option_names::kEnableSkipLayerNormStrictMode, MakeStringWithClassicLocale(info.enable_skip_layer_norm_strict_mode)},
       {cuda::provider_option_names::kPreferNHWCMode, MakeStringWithClassicLocale(info.prefer_nhwc)},
       {cuda::provider_option_names::kUseEPLevelUnifiedStream, MakeStringWithClassicLocale(info.use_ep_level_unified_stream)},
       {cuda::provider_option_names::kUseTF32, MakeStringWithClassicLocale(info.use_tf32)},
diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.h
@@ -71,7 +71,6 @@ struct CUDAExecutionProviderInfo {
 
   cuda::TunableOpInfo tunable_op{};
 
-  bool enable_skip_layer_norm_strict_mode{false};
   bool prefer_nhwc{false};
 
   bool use_ep_level_unified_stream{false};
@@ -105,7 +104,6 @@ struct std::hash<::onnxruntime::CUDAExecutionProviderInfo> {
                   (static_cast<size_t>(info.tunable_op.enable) << 24) ^
                   (static_cast<size_t>(info.tunable_op.tuning_enable) << 25) ^
                   (static_cast<size_t>(info.cudnn_conv1d_pad_to_nc1d) << 26) ^
-                  (static_cast<size_t>(info.enable_skip_layer_norm_strict_mode) << 27) ^
                   (static_cast<size_t>(info.prefer_nhwc) << 28) ^
                   (static_cast<size_t>(info.use_ep_level_unified_stream) << 29) ^
                   (static_cast<size_t>(info.use_tf32) << 30) ^
diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc
@@ -242,7 +242,6 @@ struct CUDA_Provider : Provider {
     info.tunable_op.enable = params->tunable_op_enable;
     info.tunable_op.tuning_enable = params->tunable_op_tuning_enable;
     info.tunable_op.max_tuning_duration_ms = params->tunable_op_max_tuning_duration_ms;
-    info.enable_skip_layer_norm_strict_mode = params->enable_skip_layer_norm_strict_mode != 0;
     info.use_ep_level_unified_stream = params->use_ep_level_unified_stream != 0;
     info.use_tf32 = params->use_tf32 != 0;
     info.sdpa_kernel = params->sdpa_kernel;
@@ -276,7 +275,6 @@ struct CUDA_Provider : Provider {
     cuda_options.cudnn_conv_use_max_workspace = internal_options.cudnn_conv_use_max_workspace;
     cuda_options.enable_cuda_graph = internal_options.enable_cuda_graph;
     cuda_options.cudnn_conv1d_pad_to_nc1d = internal_options.cudnn_conv1d_pad_to_nc1d;
-    cuda_options.enable_skip_layer_norm_strict_mode = internal_options.enable_skip_layer_norm_strict_mode;
     cuda_options.prefer_nhwc = internal_options.prefer_nhwc;
     cuda_options.use_ep_level_unified_stream = internal_options.use_ep_level_unified_stream;
     cuda_options.use_tf32 = internal_options.use_tf32;
diff --git a/onnxruntime/core/providers/cuda/cuda_stream_handle.cc b/onnxruntime/core/providers/cuda/cuda_stream_handle.cc
@@ -214,7 +214,9 @@ void* CudaStream::GetResource(int version, int id) const {
       return reinterpret_cast<void*>(ep_info_.cudnn_conv1d_pad_to_nc1d);
       break;
     case CudaResource::enable_skip_layer_norm_strict_mode_t:
-      return reinterpret_cast<void*>(ep_info_.enable_skip_layer_norm_strict_mode);
+      // [Deprecated] SkipLayerNorm always accumulates in fp32; the strict-mode option no longer
+      // affects computation. Kept for backward compatibility and always reported as false.
+      return reinterpret_cast<void*>(false);
       break;
     case CudaResource::prefer_nhwc_t:
       return reinterpret_cast<void*>(ep_info_.prefer_nhwc);
diff --git a/onnxruntime/core/providers/cuda/plugin/cuda_ep.cc b/onnxruntime/core/providers/cuda/plugin/cuda_ep.cc
@@ -202,7 +202,6 @@ CudaEp::CudaEp(CudaEpFactory& factory, const Config& config, const OrtLogger& lo
   // below — no function-signature change.
   onnxruntime::cuda::detail::CudaKernelAdapterRuntimeConfig adapter_config;
   adapter_config.use_tf32 = config_.use_tf32;
-  adapter_config.skip_layer_norm_strict_mode = config_.enable_skip_layer_norm_strict_mode;
   adapter_config.cudnn_conv_algo = config_.cudnn_conv_algo;
   adapter_config.cudnn_conv_use_max_workspace = config_.cudnn_conv_use_max_workspace;
   adapter_config.cudnn_conv1d_pad_to_nc1d = config_.cudnn_conv1d_pad_to_nc1d;
diff --git a/onnxruntime/core/providers/cuda/plugin/cuda_ep.h b/onnxruntime/core/providers/cuda/plugin/cuda_ep.h
@@ -24,24 +24,23 @@ class CudaEp : public onnxruntime::ep::adapter::Ep {
  public:
   /// Configuration parameters for the CUDA EP, parsed from session options.
   struct Config {
-    bool prefer_nhwc = false;                         ///< Use NHWC data layout when available.
-    bool use_tf32 = true;                             ///< Enable TF32 math on Ampere+ GPUs.
-    bool enable_skip_layer_norm_strict_mode = false;  ///< Strict mode for SkipLayerNorm kernel.
-    int device_id = 0;                                ///< CUDA device ordinal.
-    int cudnn_conv_algo = 0;                          ///< cuDNN convolution algorithm selection.
-    bool cudnn_conv_use_max_workspace = true;         ///< Use maximum workspace for cuDNN conv algo search.
-    bool cudnn_conv1d_pad_to_nc1d = false;            ///< Pad 1D convolutions to NC1D format.
-    bool fuse_conv_bias = false;                      ///< Enable cuDNN frontend conv+bias fusion.
-    int sdpa_kernel = 0;                              ///< Attention backend bitmask override.
-    bool enable_cuda_graph = false;                   ///< Enable CUDA graph capture and replay.
-    int min_num_runs_before_cuda_graph_capture = 2;   ///< Warm-up runs before graph capture begins.
-    bool has_user_compute_stream = false;             ///< Whether user provided an external CUDA stream.
-    void* user_compute_stream = nullptr;              ///< User-provided CUDA stream (cudaStream_t cast to void*).
-    bool do_copy_in_default_stream = true;            ///< Use default stream for H2D/D2H copies.
-    bool use_ep_level_unified_stream = false;         ///< Force all ops to share one stream (no concurrency).
-    void* external_alloc = nullptr;                   ///< External GPU memory allocation function pointer.
-    void* external_free = nullptr;                    ///< External GPU memory deallocation function pointer.
-    void* external_empty_cache = nullptr;             ///< External GPU memory cache-clear function pointer.
+    bool prefer_nhwc = false;                        ///< Use NHWC data layout when available.
+    bool use_tf32 = true;                            ///< Enable TF32 math on Ampere+ GPUs.
+    int device_id = 0;                               ///< CUDA device ordinal.
+    int cudnn_conv_algo = 0;                         ///< cuDNN convolution algorithm selection.
+    bool cudnn_conv_use_max_workspace = true;        ///< Use maximum workspace for cuDNN conv algo search.
+    bool cudnn_conv1d_pad_to_nc1d = false;           ///< Pad 1D convolutions to NC1D format.
+    bool fuse_conv_bias = false;                     ///< Enable cuDNN frontend conv+bias fusion.
+    int sdpa_kernel = 0;                             ///< Attention backend bitmask override.
+    bool enable_cuda_graph = false;                  ///< Enable CUDA graph capture and replay.
+    int min_num_runs_before_cuda_graph_capture = 2;  ///< Warm-up runs before graph capture begins.
+    bool has_user_compute_stream = false;            ///< Whether user provided an external CUDA stream.
+    void* user_compute_stream = nullptr;             ///< User-provided CUDA stream (cudaStream_t cast to void*).
+    bool do_copy_in_default_stream = true;           ///< Use default stream for H2D/D2H copies.
+    bool use_ep_level_unified_stream = false;        ///< Force all ops to share one stream (no concurrency).
+    void* external_alloc = nullptr;                  ///< External GPU memory allocation function pointer.
+    void* external_free = nullptr;                   ///< External GPU memory deallocation function pointer.
+    void* external_empty_cache = nullptr;            ///< External GPU memory cache-clear function pointer.
   };
 
   CudaEp(CudaEpFactory& factory, const Config& config, const OrtLogger& logger);
diff --git a/onnxruntime/core/providers/cuda/plugin/cuda_ep_factory.cc b/onnxruntime/core/providers/cuda/plugin/cuda_ep_factory.cc
@@ -482,7 +482,6 @@ OrtStatus* ORT_API_CALL CudaEpFactory::CreateEpImpl(
   const std::string prefer_nhwc_key = ep_options_prefix + "prefer_nhwc";
   const std::string prefer_nhwc_layout_key = ep_options_prefix + "prefer_nhwc_layout";
   const std::string use_tf32_key = ep_options_prefix + "use_tf32";
-  const std::string skip_layer_norm_key = ep_options_prefix + "enable_skip_layer_norm_strict_mode";
   const std::string cudnn_use_max_workspace_key = ep_options_prefix + "cudnn_conv_use_max_workspace";
   const std::string cudnn_conv1d_pad_key = ep_options_prefix + "cudnn_conv1d_pad_to_nc1d";
   const std::string cudnn_conv_algo_key = ep_options_prefix + "cudnn_conv_algo";
@@ -505,9 +504,6 @@ OrtStatus* ORT_API_CALL CudaEpFactory::CreateEpImpl(
       {prefer_nhwc_key, prefer_nhwc_layout_key, "ep.cuda.prefer_nhwc_layout", "prefer_nhwc", "prefer_nhwc_layout"},
       config.prefer_nhwc);
   read_session_config_bool({use_tf32_key, "ep.cuda.use_tf32", "use_tf32"}, config.use_tf32);
-  read_session_config_bool(
-      {skip_layer_norm_key, "ep.cuda.enable_skip_layer_norm_strict_mode", "enable_skip_layer_norm_strict_mode"},
-      config.enable_skip_layer_norm_strict_mode);
   read_session_config_bool(
       {cudnn_use_max_workspace_key, "ep.cuda.cudnn_conv_use_max_workspace", "cudnn_conv_use_max_workspace"},
       config.cudnn_conv_use_max_workspace);
diff --git a/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h b/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h
diff --git a/onnxruntime/test/contrib_ops/skiplayernorm_op_test.cc b/onnxruntime/test/contrib_ops/skiplayernorm_op_test.cc
