[GPU] add oneDNN SDPA primitive as a debug feature

src/plugins/intel_gpu/include/intel_gpu/runtime/internal_properties.hpp

@@ -134,6 +134,7 @@ static constexpr Property<std::string, PropertyMutability::RW> config_file{"CONF
 static constexpr Property<float, PropertyMutability::RW> buffers_preallocation_ratio{"GPU_BUFFERS_PREALLOCATION_RATIO"};
 static constexpr Property<size_t, PropertyMutability::RW> max_kernels_per_batch{"GPU_MAX_KERNELS_PER_BATCH"};
 static constexpr Property<bool, PropertyMutability::RW> use_onednn{"GPU_USE_ONEDNN"};
+static constexpr Property<bool, PropertyMutability::RW> disable_sdpa_micro{"GPU_DISABLE_SDPA_MICRO"};
 static constexpr Property<bool, PropertyMutability::RW> use_cm{"GPU_USE_CM"};
 
 static constexpr Property<bool, ov::PropertyMutability::RW> help{"HELP"};
@@ -142,6 +143,7 @@ static constexpr Property<bool, ov::PropertyMutability::RW> verbose_color{"VERBO
 static constexpr Property<std::string, ov::PropertyMutability::RW> debug_config{"GPU_DEBUG_CONFIG"};
 static constexpr Property<std::string, ov::PropertyMutability::RW> log_to_file{"GPU_LOG_TO_FILE"};
 static constexpr Property<bool, ov::PropertyMutability::RW> disable_usm{"GPU_DISABLE_USM"};
+static constexpr Property<bool, ov::PropertyMutability::RW> use_onednn_sdpa{"GPU_USE_ONEDNN_SDPA"};
 static constexpr Property<bool, ov::PropertyMutability::RW> disable_onednn_post_ops_opt{"GPU_DISABLE_ONEDNN_POST_OPS_OPT"};
 static constexpr Property<std::string, PropertyMutability::RW> dump_graphs_path{"GPU_DUMP_GRAPHS_PATH"};
 static constexpr Property<std::string, ov::PropertyMutability::RW> dump_profiling_data_path{"GPU_DUMP_PROFILING_DATA_PATH"};

src/plugins/intel_gpu/include/intel_gpu/runtime/options.inl

@@ -52,6 +52,7 @@ OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, force_implementations, ImplForc
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, partial_build_program, false, "Early exit from model compilation process which allows faster execution graph dumping")
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, allow_new_shape_infer, false, "Switch between new and old shape inference flow. Shall be removed soon")
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, use_onednn, false, "Enable/Disable onednn for usage for particular model/platform")
+OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, disable_sdpa_micro, false, "Disable oneDNN-derived SDPA micro kernel and fall back to the OCL sdpa_opt kernel")
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, use_cm, true, "Enable/Disable CM for usage for particular model/platform")
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, max_kernels_per_batch, 8, "Controls how many kernels we combine into batch for more efficient ocl compilation")
 OV_CONFIG_RELEASE_INTERNAL_OPTION(ov::intel_gpu, impls_cache_capacity, 300, "Controls capacity of LRU implementations cache that is created for each program object for dynamic models")
@@ -73,6 +74,7 @@ OV_CONFIG_DEBUG_GLOBAL_OPTION(ov::intel_gpu, dump_profiling_data_per_iter, false
 OV_CONFIG_DEBUG_GLOBAL_OPTION(ov::intel_gpu, log_to_file, "", "Save verbose log to specified file")
 OV_CONFIG_DEBUG_GLOBAL_OPTION(ov::intel_gpu, debug_config, "", "Path to debug config in json format")
 
+OV_CONFIG_DEBUG_OPTION(ov::intel_gpu, use_onednn_sdpa, false, "Use oneDNN SDPA primitive implementation for debugging")
 OV_CONFIG_DEBUG_OPTION(ov::intel_gpu, disable_onednn_post_ops_opt, false, "Disable optimization pass for onednn post-ops")
 OV_CONFIG_DEBUG_OPTION(ov::intel_gpu, dump_profiling_data_path, "", "Save csv file with per-stage and per-primitive profiling data to specified folder")
 OV_CONFIG_DEBUG_OPTION(ov::intel_gpu, average_counters, "", "Save csv file with per-primitive averaged execution time. Output format matches benchmark_app --report_type average_counters and OV_CPU_AVERAGE_COUNTERS")

src/plugins/intel_gpu/src/graph/impls/ocl_v2/sdpa/paged_attention_opt.cpp

@@ -1361,6 +1361,14 @@ class PagedAttentionOptImpl : public SDPAImplBase {
         auto& engine = params.get_program().get_engine();
         const auto desc = params.typed_desc<paged_attention>();
 
+        // Knob (OV_GPU_DISABLE_SDPA_MICRO=1): force PagedAttention onto the OCL sdpa_opt path
+        // instead of the oneDNN-derived micro kernel, for accuracy A/B without rebuilding.
+        if (params.get_program().get_config().get_disable_sdpa_micro()) {
+            GPU_DEBUG_TRACE_DETAIL << "[GPU] PA micro SDPA kernel DISABLED via OV_GPU_DISABLE_SDPA_MICRO; "
+                                      "falling back to OCL paged_attention/sdpa_opt kernel.\n";
+            return false;
+        }
+
         if (params.get_device_info().supports_immad) {
             const auto supports_microkernels = cldnn::query_microkernels_supported(engine, params.get_program().get_config());
             if (params.get_device_info().arch < gpu_arch::xe_hpg || !supports_microkernels) {

src/plugins/intel_gpu/src/graph/impls/ocl_v2/sdpa/sdpa_gen_micro.cpp

@@ -18,6 +18,10 @@
 #include "paged_attention_opt.hpp"
 #include "sdpa_base.hpp"
 #include "../utils/kernel_generator.hpp"
+
+#include <set>
+#include <sstream>
+#include <string>
 // clang-format on
 namespace ov::intel_gpu::ocl {
 namespace {
@@ -437,12 +441,40 @@ sdpa_config_t xe2_q_h256_s768_2nd_integrated = {64, 16, 16, 16, 16, 1, 16, 1};
 sdpa_config_t xe2_q_h256_s512_2nd_integrated = {32, 32, 32, 16, 16, 1, 8, 2};
 sdpa_config_t xe2_q_h256_s384_2nd_integrated = {16, 16, 16, 16, 16, 1, 16, 1};
 
-sdpa_config_t xe3_h128 = {32, 16, 32, 16, 16, 2, 16, 2};
-sdpa_config_t xe3_h256 = {32, 16, 32, 16, 16, 2, 16, 2};
-
-sdpa_config_t xe3_h512 = {32, 16, 32, 16, 16, 2, 16, 2};
-sdpa_config_t xe3_h512_2nd = {32, 16, 32, 16, 16, 1, 16, 1};
-sdpa_config_t xe3_q_h512_2nd = {32, 16, 32, 16, 16, 1, 16, 1};
+// xe3p configs synced from oneDNN tuned table
+// (thirdparty/onednn_gpu/src/gpu/intel/sdpa/configs.cpp, gpu_arch_t::xe3p entries).
+// ALL oneDNN xe3p rows are represented here, INCLUDING the ones oneDNN tags with the
+// `fma` property. fma is oneDNN-internal (it selects the non-systolic GEMM path); the
+// OV micro SDPA path has no fma notion and choose_config_xe3p has no fma argument, so
+// we IGNORE the fma tag and just reuse those rows' tuned tile configs. (We still drop
+// the `f32` rows: those stay gated on the f32 property, which OV f16 SDPA never sets.)
+// choose_config_xe3p() below reproduces oneDNN's head_size CEILING match + exact
+// second_token/quantized match + xe3p->xe2 fallback over the fma-folded row set.
+//
+// Verbatim oneDNN xe3p rows (fma tag ignored; config = {unroll_m_kq,unroll_n_kq,
+//   unroll_m_vs,unroll_n_vs, wg_m_kq,wg_n_kq,wg_m_vs,wg_n_vs}):
+//   {xe3p, 32, fma}                              -> xe3p_h32
+//   {xe3p, 32, fma|2nd|quant}                    -> xe3p_h32 (same config)
+//   {xe3p, 64, fma|2nd}  {xe3p,64,fma|2nd|quant} -> xe3p_h64_2nd
+//   {xe3p,128}           {xe3p,128,quant}        -> xe3p_h128
+//   {xe3p,128,2nd}       {xe3p,128,2nd|quant}    -> xe3p_h128_2nd
+//   {xe3p,256,2nd}       {xe3p,256,2nd|quant}    -> xe3p_h256_2nd
+//   {xe3p,256,quant}                             -> xe3p_q_h256
+//   {xe3p,512}                                   -> xe3p_h512
+//   {xe3p,512,2nd}                               -> xe3p_h512_2nd
+//   {xe3p,512,2nd|quant} {xe3p,512,fma|quant}    -> xe3p_q_h512_2nd (same config)
+sdpa_config_t xe3p_h32 = {16, 16, 16, 16, 2, 2, 2, 2};
+sdpa_config_t xe3p_h64_2nd = {16, 16, 16, 16, 8, 4, 8, 4};
+
+sdpa_config_t xe3p_h128 = {32, 32, 32, 32, 4, 4, 4, 4};
+sdpa_config_t xe3p_h128_2nd = {32, 32, 32, 32, 4, 1, 4, 1};
+
+sdpa_config_t xe3p_h256_2nd = {32, 32, 32, 32, 8, 1, 8, 1};
+sdpa_config_t xe3p_q_h256 = {32, 32, 32, 32, 8, 2, 8, 2};
+
+sdpa_config_t xe3p_h512 = {32, 16, 32, 16, 16, 2, 16, 2};
+sdpa_config_t xe3p_h512_2nd = {32, 16, 32, 16, 16, 1, 16, 1};
+sdpa_config_t xe3p_q_h512_2nd = {32, 16, 32, 16, 16, 1, 16, 1};
 
 sdpa_config_t* choose_config_xehpg(int head_size, int seq, bool thin_q, bool quantized, bool is_pa, bool is_prefill) {
     if (head_size <= 32) {
@@ -794,22 +826,67 @@ sdpa_config_t* choose_config_xe2(int head_size, int seq, bool thin_q, bool quant
     return choose_config_xehpc(head_size, seq, thin_q, quantized, is_integrated, is_pa, is_prefill);
 }
 sdpa_config_t* choose_config_xe3p(int head_size, int seq, bool thin_q, bool quantized, bool is_integrated, bool is_pa, bool is_prefill) {
+    // Faithful port of oneDNN's data-driven choose_config for gpu_arch_t::xe3p
+    // (thirdparty/onednn_gpu/src/gpu/intel/sdpa/configs.cpp). oneDNN does a CEILING
+    // match on head_size (query.head_size <= key.head_size, smallest bucket that
+    // fits), exact match on the second_token/quantized properties, then falls back
+    // xe3p -> xe3 -> xe2 (there are no xe3 rows, so a miss lands on the xe2 table =
+    // choose_config_xe2 here).
+    //
+    // !!! fma is IGNORED (folded in, not dropped): oneDNN tags some xe3p rows with
+    // the `fma` property (its non-systolic GEMM path), but the OV micro SDPA path
+    // has no fma notion and choose_config_xe3p has no fma argument. Rather than let
+    // those rows go unmatched, we reuse their tuned tile configs as the xe3p config
+    // for that (head_size, thin_q, quantized) bucket. (The `f32`-tagged rows ARE
+    // still skipped -- OV f16 SDPA never sets f32.) This is why head_size 32/64 now
+    // have their own configs instead of falling through to h128.
+    //
+    // Resolved per (head_size ceiling bucket, thin_q, quantized), fma folded:
+    //   hs<=32  : plain -> h32 ; quant -> h128 (no fma|quant-plain row -> ceiling
+    //             to {xe3p,128,quant}) ; 2nd -> h64_2nd (no fma|2nd-plain hs32 row
+    //             -> ceiling to {xe3p,64,fma|2nd}) ; 2nd|quant -> h32
+    //   hs<=64  : plain -> h128 (no hs64 plain row -> ceiling to {xe3p,128}) ;
+    //             quant -> h128 ; 2nd/2nd|quant -> h64_2nd
+    //   hs<=128 : plain/quant -> h128 ; 2nd/2nd|quant -> h128_2nd
+    //   hs<=256 : quant -> q_h256 ; 2nd/2nd|quant -> h256_2nd ;
+    //             plain has NO xe3p-256 row -> ceiling to {xe3p,512} plain (h512)
+    //   hs<=512 : plain -> h512 ; 2nd -> h512_2nd ; quant|2nd / plain|quant
+    //             (via {xe3p,512,fma|quant}) -> q_h512_2nd
+    //   hs>512  : no xe3p row -> fall back to xe2
+	// xe3p tuning table is only for non-PA SDPA; PA uses the xe2 fallback below because
+	// xe3p-specific PA configs have not been validated yet.
+	if(!is_pa) {
+    if (head_size <= 32) {
+        if (thin_q)
+            return quantized ? &xe3p_h32 : &xe3p_h64_2nd;
+        return quantized ? &xe3p_h128 : &xe3p_h32;
+    }
+    if (head_size <= 64) {
+        if (thin_q)
+            return &xe3p_h64_2nd;  // {xe3p,64, fma|2nd} and {fma|2nd|quant}
+        return &xe3p_h128;         // no {xe3p,64} plain/quant -> ceiling to {xe3p,128}
+    }
     if (head_size <= 128) {
-        return &xe3_h128;
+        if (thin_q)
+            return &xe3p_h128_2nd;  // {xe3p,128, 2nd} and {2nd|quant}
+        return &xe3p_h128;          // {xe3p,128} and {quant}
     }
     if (head_size <= 256) {
-        return &xe3_h256;
-        return choose_config_xe2(head_size, seq, thin_q, quantized, is_integrated, is_pa, is_prefill);
+        if (thin_q)
+            return &xe3p_h256_2nd;  // {xe3p,256, 2nd} and {2nd|quant}
+        if (quantized)
+            return &xe3p_q_h256;    // {xe3p,256, quant}
+        // No {xe3p,256} plain row -> head_size ceiling matches {xe3p,512} plain.
+        return &xe3p_h512;
     }
     if (head_size <= 512) {
-        if (thin_q) {
-            if (quantized) {
-                return &xe3_q_h512_2nd;
-            }
-            return &xe3_h512_2nd;
-        }
-        return &xe3_h512;
+        if (thin_q)
+            return quantized ? &xe3p_q_h512_2nd : &xe3p_h512_2nd;  // {512,2nd}/{512,2nd|quant}
+        if (quantized)
+            return &xe3p_q_h512_2nd;  // {xe3p,512, fma|quant} (fma folded; same config)
+        return &xe3p_h512;            // {xe3p,512} plain
     }
+	}
     return choose_config_xe2(head_size, seq, thin_q, quantized, is_integrated, is_pa, is_prefill);
 }
 
@@ -1544,6 +1621,19 @@ void SDPAMicroGenerator::init_microkernels(const kernel_impl_params& params,
 
     OPENVINO_ASSERT(config != nullptr);
 
+    GPU_DEBUG_IF(ExecutionConfig::get_verbose() >= static_cast<std::underlying_type_t<LogLevel>>(LogLevel::TRACE_DETAIL)) {
+        std::ostringstream oss;
+        oss << "[choose_config arch=" << static_cast<int>(device_info.arch) << "] head_size=" << static_cast<int32_t>(k_head_size)
+            << " seq=" << nkeys_v << " thin_q=" << thin_q << " quantized=" << is_quantized << " is_integrated=" << is_integrated
+            << " is_pa=" << is_paged_attention << " is_prefill=" << is_prefill << " is_gqa_single_token=" << is_gqa_single_token
+            << " => config={" << config->unroll_m_kq << "," << config->unroll_n_kq << "," << config->unroll_m_vs << ","
+            << config->unroll_n_vs << "," << config->wg_m_kq << "," << config->wg_n_kq << "," << config->wg_m_vs << ","
+            << config->wg_n_vs << "}";
+        static std::set<std::string> seen_choose_config;
+        if (seen_choose_config.insert(oss.str()).second)
+            GPU_DEBUG_TRACE_DETAIL << oss.str() << "\n";
+    }
+
     /* Get device information */
     micro::HWInformation hw_info;
     hw_info.euCount = device_info.execution_units_count;
@@ -1828,6 +1918,48 @@ void SDPAMicroGenerator::init_microkernels(const kernel_impl_params& params,
         throw;
     }
 
+    // [selected microkernel dump] The 8-field choose_config struct is only a TUNING HINT; the ACTUAL
+    // generated microkernel is what select_gemm_microkernel() resolved (it can differ between models that
+    // share the same config struct -- e.g. phi head_size=96 and llama head_size=128 both resolve to
+    // xe3p_h128 {32,32,32,32,4,4,4,4} yet build DIFFERENT kernels because k_head_size/d_max/problem sizes
+    // differ). This logs the per-GEMM selected strategy (Package settings) so we can see what really runs.
+    // Deduped to one line per distinct tuple (called per-layer-per-iter); fn runs under lock `m`.
+    GPU_DEBUG_IF(ExecutionConfig::get_verbose() >= static_cast<std::underlying_type_t<LogLevel>>(LogLevel::TRACE_DETAIL)) {
+        // getSetting() throws on an unknown key -- wrap it so a missing setting prints -1 instead of aborting.
+        auto setting_or = [](const micro::Package& pkg, const char* name) -> int {
+            try {
+                return pkg.getSetting(name);
+            } catch (const std::runtime_error&) {
+                return -1;
+            }
+        };
+        // micro::Type encodes its value as a bitfield (raw int is unreadable) -> print bit width + kind.
+        auto fmt_type = [](micro::Type t) -> std::string {
+            std::string k = t.is4() ? "i4" : (t.isInteger() ? "int" : "fp");
+            return std::to_string(t.bits()) + (t.isInteger() && !t.isSigned() ? "u" : "") + k;
+        };
+        auto fmt_pkg = [&](std::ostringstream& o, const micro::Package& pkg) {
+            o << "sg_tile=" << setting_or(pkg, "sg_tile_m") << "x" << setting_or(pkg, "sg_tile_n")
+              << " wg_tile=" << setting_or(pkg, "wg_tile_m") << "x" << setting_or(pkg, "wg_tile_n")
+              << " sg_per_wg=" << setting_or(pkg, "sg_per_wg_m") << "x" << setting_or(pkg, "sg_per_wg_n")
+              << "x" << setting_or(pkg, "sg_per_wg_k") << " slm=" << setting_or(pkg, "slm_size")
+              << " systolic=" << pkg.systolic << " grfMin=" << pkg.grfMin << " barriers=" << pkg.barrierCount;
+        };
+        std::ostringstream oss;
+        oss << "[micro_selected arch=" << static_cast<int>(device_info.arch) << "] head_size=" << static_cast<int32_t>(k_head_size)
+            << " v_head_size=" << static_cast<int32_t>(v_head_size) << " d_max=" << d_max
+            << " is_pa=" << is_paged_attention << " is_prefill=" << is_prefill << " thin_q=" << thin_q
+            << " quantized=" << is_quantized
+            << " | KQ{Ta_ext=" << fmt_type(problem_kq.Ta_ext) << " Tb_ext=" << fmt_type(problem_kq.Tb_ext) << " ";
+        fmt_pkg(oss, gemm_kq);
+        oss << "} | VS{Ta_ext=" << fmt_type(problem_vs.Ta_ext) << " ";
+        fmt_pkg(oss, gemm_vs);
+        oss << "}";
+        static std::set<std::string> seen_micro_selected;
+        if (seen_micro_selected.insert(oss.str()).second)
+            GPU_DEBUG_TRACE_DETAIL << oss.str() << "\n";
+    }
+
     if (!is_prefill && !is_gqa_single_token) {
         /* Update for optional GEMM: Vc*S */
         opts_vs.scaleA = false;

src/plugins/intel_gpu/src/graph/impls/ocl_v2/sdpa/sdpa_opt.cpp

@@ -10,6 +10,7 @@
 
 #include "sdpa_opt.hpp"
 
+
 #include "../primitive_ocl_base.hpp"
 #include "../utils/kernel_generator.hpp"
 #include "common_utils/jitter.hpp"
@@ -179,6 +180,14 @@ class SDPAOptImpl : public SDPAImplBase {
 
 bool SDPAOpt::supports_micro_sdpa(const RuntimeParams& params) {
 #ifdef ENABLE_ONEDNN_FOR_GPU
+    // Knob (OV_GPU_DISABLE_SDPA_MICRO=1): force the SDPA backend onto the OCL sdpa_opt kernel
+    // instead of the oneDNN-derived micro kernel, for accuracy A/B without rebuilding.
+    if (params.get_program().get_config().get_disable_sdpa_micro()) {
+        GPU_DEBUG_TRACE_DETAIL << "[GPU] SDPA micro kernel DISABLED via OV_GPU_DISABLE_SDPA_MICRO; "
+                                  "falling back to OCL sdpa_opt kernel.\n";
+        return false;
+    }
+
     auto& engine = params.get_program().get_engine();
     const auto& device_info = engine.get_device_info();
     auto desc = params.typed_desc<scaled_dot_product_attention>();