From cf9e9ad47abea37a4206e9b36ee2a8cdc5659a30 Mon Sep 17 00:00:00 2001 From: Beheshti Date: Wed, 8 Jul 2026 10:26:55 -0700 Subject: [PATCH 01/15] apply GPU transformation to avoid KV cache broadcast --- ...ate_reshape_concat_reshape_sdpa_fusion.cpp | 132 ++++++++++++++++++ ...ate_reshape_concat_reshape_sdpa_fusion.hpp | 17 +++ .../src/plugin/transformations_pipeline.cpp | 3 + 3 files changed, 152 insertions(+) create mode 100644 src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.cpp create mode 100644 src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp diff --git a/src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.cpp b/src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.cpp new file mode 100644 index 00000000000000..443111ed344046 --- /dev/null +++ b/src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.cpp @@ -0,0 +1,132 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp" + +#include +#include +#include + +#include "openvino/core/partial_shape.hpp" +#include "openvino/core/graph_util.hpp" +#include "openvino/core/shape.hpp" +#include "openvino/core/validation_util.hpp" +#include "openvino/op/concat.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/scatter_update.hpp" +#include "openvino/pass/pattern/matcher.hpp" +#include "openvino/pass/pattern/op/or.hpp" +#include "openvino/pass/pattern/op/predicate.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" + +#include "transformations/utils/utils.hpp" + +#include "intel_gpu/op/kv_cache.hpp" +#include "intel_gpu/op/sdpa.hpp" +#include + +namespace ov::intel_gpu { +ScatterUpdateReshapeConcatReshapeSDPAFusion::ScatterUpdateReshapeConcatReshapeSDPAFusion() { + using namespace ov::pass::pattern; + + auto input_a_m = any_input(); + auto input_attn_mask_m = any_input(); + auto input_scale_m = any_input(); + + auto past_param_key_m = wrap_type(); + auto indices_key_m = any_input(); + auto updates_key_m = any_input(); + auto axis_key_m = wrap_type(); + + auto scatter_update_key_m = wrap_type({past_param_key_m, indices_key_m, updates_key_m, axis_key_m}); + + auto vsplit_axis_key_m = any_input(); + auto vsplit_lengths_key_m = any_input(); + auto variadic_split_key_m = wrap_type( + {scatter_update_key_m, vsplit_axis_key_m, vsplit_lengths_key_m}); + + auto scatter_or_vsplit_key_m = std::make_shared( + OutputVector{scatter_update_key_m, variadic_split_key_m}); + + auto reshape1_pattern_key_m = any_input(); + auto reshape_4d_to_5d_key_m = wrap_type({scatter_or_vsplit_key_m, reshape1_pattern_key_m}); + + auto concat_key_m = wrap_type({reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m}); + + auto reshape2_pattern_key_m = any_input(); + auto reshape_5d_to_4d_key_m = wrap_type({concat_key_m, reshape2_pattern_key_m}); + + auto past_param_value_m = wrap_type(); + auto indices_value_m = any_input(); + auto updates_value_m = any_input(); + auto axis_value_m = wrap_type(); + + auto scatter_update_value_m = wrap_type({past_param_value_m, indices_value_m, updates_value_m, axis_value_m}); + + auto vsplit_axis_value_m = any_input(); + auto vsplit_lengths_value_m = any_input(); + auto variadic_split_value_m = wrap_type( + { scatter_update_value_m, vsplit_axis_value_m, vsplit_lengths_value_m }); + + auto scatter_or_vsplit_value_m = std::make_shared( + OutputVector{ scatter_update_value_m, variadic_split_value_m }); + + auto reshape1_pattern_value_m = any_input(); + auto reshape_4d_to_5d_value_m = wrap_type({scatter_or_vsplit_value_m, reshape1_pattern_value_m}); + + auto concat_value_m = wrap_type({reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m}); + + auto reshape2_pattern_value_m = any_input(); + auto reshape_5d_to_4d_value_m = wrap_type({concat_value_m, reshape2_pattern_value_m}); + + auto sdpa_without_attn_mask_m = wrap_type({input_a_m, reshape_5d_to_4d_key_m, reshape_5d_to_4d_value_m}); + auto sdpa_with_attn_mask_m = + wrap_type({input_a_m, reshape_5d_to_4d_key_m, reshape_5d_to_4d_value_m, any_input()}); + auto sdpa_with_attn_mask_and_scale_m = + wrap_type({input_a_m, reshape_5d_to_4d_key_m, reshape_5d_to_4d_value_m, input_attn_mask_m, input_scale_m}); + + + auto sdpa_m = std::make_shared(OutputVector{sdpa_without_attn_mask_m, sdpa_with_attn_mask_m, sdpa_with_attn_mask_and_scale_m}); + + ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](ov::pass::pattern::Matcher& m) { + const auto& pattern_map = m.get_pattern_value_map(); + + std::shared_ptr sdpa; + sdpa = std::dynamic_pointer_cast( + pattern_map.at(sdpa_m).get_node_shared_ptr()); + const auto past_param_key = + std::dynamic_pointer_cast(pattern_map.at(past_param_key_m).get_node_shared_ptr()); + const auto past_param_value = + std::dynamic_pointer_cast(pattern_map.at(past_param_value_m).get_node_shared_ptr()); + + + OutputVector data_inputs; + + if (pattern_map.count(scatter_update_key_m) > 0) { + const auto scatter_update_key = + std::dynamic_pointer_cast(pattern_map.at(scatter_update_key_m).get_node_shared_ptr()); + } if (pattern_map.count(variadic_split_key_m) > 0) { + const auto variadic_split_key = + std::dynamic_pointer_cast(pattern_map.at(variadic_split_key_m).get_node_shared_ptr()); + sdpa->input(1).replace_source_output(variadic_split_key->output(1)); + } + if (pattern_map.count(scatter_update_value_m) > 0) { + const auto scatter_update_value = + std::dynamic_pointer_cast(pattern_map.at(scatter_update_value_m).get_node_shared_ptr()); + } if (pattern_map.count(variadic_split_value_m) > 0) { + const auto variadic_split_value = + std::dynamic_pointer_cast(pattern_map.at(variadic_split_value_m).get_node_shared_ptr()); + sdpa->input(2).replace_source_output(variadic_split_value->output(1)); + + } + return true; + }; + auto m = std::make_shared(sdpa_m, "ScatterUpdateReshapeConcatReshapeSDPAFusion"); + + this->register_matcher(m, callback); +} + +} // namespace ov::intel_gpu \ No newline at end of file diff --git a/src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp b/src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp new file mode 100644 index 00000000000000..dd096f9ede547b --- /dev/null +++ b/src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp @@ -0,0 +1,17 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" + +namespace ov::intel_gpu { + +class ScatterUpdateReshapeConcatReshapeSDPAFusion : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("ScatterUpdateReshapeConcatReshapeSDPAFusion"); + ScatterUpdateReshapeConcatReshapeSDPAFusion(); +}; + +} // namespace ov::intel_gpu diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index c626f8c876cb6c..9ad5a2cba9cbf6 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -114,6 +114,7 @@ #include "plugin/transformations/transpose_fusion.hpp" #include "plugin/transformations/unsqueeze_broadcast_reshape_matmul_fusion.hpp" #include "plugin/transformations/unsqueeze_broadcast_reshape_sdpa_fusion.hpp" +#include "plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp" #include "plugin/transformations/disable_fp16_comp_rms.hpp" #include "plugin/transformations/swiglu_fusion_with_clamp.hpp" #include "plugin/transformations/disable_fp16_comp_cumsum_sin_gen.hpp" @@ -1609,6 +1610,8 @@ void TransformationsPipeline::apply(std::shared_ptr func) { manager.register_pass(); } manager.register_pass(); + manager.register_pass(); + manager.register_pass(); manager.register_pass(); From 0b8f545068238a929e66231afdaf15abae092800 Mon Sep 17 00:00:00 2001 From: Beheshti Date: Fri, 10 Jul 2026 12:43:46 -0700 Subject: [PATCH 02/15] apply reviewer comments --- .../GQA_decomposition_fusion.cpp | 81 +++++++++++ ...usion.hpp => GQA_decomposition_fusion.hpp} | 6 +- ...ate_reshape_concat_reshape_sdpa_fusion.cpp | 132 ------------------ .../src/plugin/transformations_pipeline.cpp | 4 +- 4 files changed, 86 insertions(+), 137 deletions(-) create mode 100644 src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp rename src/plugins/intel_gpu/src/plugin/transformations/{scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp => GQA_decomposition_fusion.hpp} (50%) delete mode 100644 src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.cpp diff --git a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp new file mode 100644 index 00000000000000..ddba56032a33b8 --- /dev/null +++ b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp @@ -0,0 +1,81 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "GQA_Decompositionfusion_fusion.hpp" +#include "openvino/pass/pattern/matcher.hpp" +#include "openvino/pass/pattern/op/or.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/concat.hpp" +#include "openvino/core/partial_shape.hpp" + +namespace ov::intel_gpu { + GQADecompositionfusion::GQADecompositionfusion() { + using namespace ov::pass::pattern; + + auto reshape_4d_to_5d = [](const ov::Output& output) { + auto in_ps = output.get_node()->get_input_partial_shape(0); + auto out_ps = output.get_node()->get_output_partial_shape(0); + return in_ps.rank().is_static() && out_ps.rank().is_static() && in_ps.size() == 4 && out_ps.size() == 5; + }; + + auto reshape_5d_to_4d = [](const ov::Output& output) { + auto in_ps = output.get_node()->get_input_partial_shape(0); + auto out_ps = output.get_node()->get_output_partial_shape(0); + return in_ps.rank().is_static() && out_ps.rank().is_static() && in_ps.size() == 5 && out_ps.size() == 4; + }; + + auto input_a_m = any_input(); + auto input_attn_mask_m = any_input(); + auto input_scale_m = any_input(); + + auto in_key = any_input(); + auto reshape1_pattern_key_m = any_input(); + auto reshape_4d_to_5d_key_m = wrap_type({ in_key, reshape1_pattern_key_m }, reshape_4d_to_5d); + + auto concat_key_m = wrap_type({ reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m }); + + auto reshape2_pattern_key_m = any_input(); + auto reshape_5d_to_4d_key_m = wrap_type({ concat_key_m, reshape2_pattern_key_m }, reshape_5d_to_4d); + + auto in_value = any_input(); + auto reshape1_pattern_value_m = any_input(); + auto reshape_4d_to_5d_value_m = wrap_type({ in_value, reshape1_pattern_value_m }); + + auto concat_value_m = wrap_type({ reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m }); + + auto reshape2_pattern_value_m = any_input(); + auto reshape_5d_to_4d_value_m = wrap_type({ concat_value_m, reshape2_pattern_value_m }); + + auto sdpa_without_attn_mask_m = wrap_type({ input_a_m, reshape_5d_to_4d_key_m, reshape_5d_to_4d_value_m }); + auto sdpa_with_attn_mask_m = + wrap_type({ input_a_m, reshape_5d_to_4d_key_m, reshape_5d_to_4d_value_m, any_input() }); + auto sdpa_with_attn_mask_and_scale_m = + wrap_type({ input_a_m, reshape_5d_to_4d_key_m, reshape_5d_to_4d_value_m, input_attn_mask_m, input_scale_m }); + + auto sdpa_m = std::make_shared(OutputVector{ sdpa_without_attn_mask_m, sdpa_with_attn_mask_m, sdpa_with_attn_mask_and_scale_m }); + + ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](ov::pass::pattern::Matcher& m) { + const auto& pattern_map = m.get_pattern_value_map(); + + auto sdpa = ov::as_type_ptr(m.get_match_root()); + + if (pattern_map.count(in_key) > 0) { + const auto key = pattern_map.at(in_key).get_node_shared_ptr(); + sdpa->input(1).replace_source_output(key->output(1)); + } + + if (pattern_map.count(in_value) > 0) { + const auto value = pattern_map.at(in_value).get_node_shared_ptr(); + sdpa->input(2).replace_source_output(value->output(1)); + + } + return true; + }; + auto m = std::make_shared(sdpa_m, "ScatterUpdateReshapeConcatReshapeSDPAFusion"); + + this->register_matcher(m, callback); + } + +} // namespace ov::intel_gpu \ No newline at end of file diff --git a/src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.hpp similarity index 50% rename from src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp rename to src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.hpp index dd096f9ede547b..8bf222a0cd95e1 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.hpp @@ -8,10 +8,10 @@ namespace ov::intel_gpu { -class ScatterUpdateReshapeConcatReshapeSDPAFusion : public ov::pass::MatcherPass { +class GQADecompositionfusion : public ov::pass::MatcherPass { public: - OPENVINO_MATCHER_PASS_RTTI("ScatterUpdateReshapeConcatReshapeSDPAFusion"); - ScatterUpdateReshapeConcatReshapeSDPAFusion(); + OPENVINO_MATCHER_PASS_RTTI("GQADecompositionfusion"); + GQADecompositionfusion(); }; } // namespace ov::intel_gpu diff --git a/src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.cpp b/src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.cpp deleted file mode 100644 index 443111ed344046..00000000000000 --- a/src/plugins/intel_gpu/src/plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.cpp +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (C) 2018-2026 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp" - -#include -#include -#include - -#include "openvino/core/partial_shape.hpp" -#include "openvino/core/graph_util.hpp" -#include "openvino/core/shape.hpp" -#include "openvino/core/validation_util.hpp" -#include "openvino/op/concat.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/parameter.hpp" -#include "openvino/op/reshape.hpp" -#include "openvino/op/scatter_update.hpp" -#include "openvino/pass/pattern/matcher.hpp" -#include "openvino/pass/pattern/op/or.hpp" -#include "openvino/pass/pattern/op/predicate.hpp" -#include "openvino/pass/pattern/op/wrap_type.hpp" - -#include "transformations/utils/utils.hpp" - -#include "intel_gpu/op/kv_cache.hpp" -#include "intel_gpu/op/sdpa.hpp" -#include - -namespace ov::intel_gpu { -ScatterUpdateReshapeConcatReshapeSDPAFusion::ScatterUpdateReshapeConcatReshapeSDPAFusion() { - using namespace ov::pass::pattern; - - auto input_a_m = any_input(); - auto input_attn_mask_m = any_input(); - auto input_scale_m = any_input(); - - auto past_param_key_m = wrap_type(); - auto indices_key_m = any_input(); - auto updates_key_m = any_input(); - auto axis_key_m = wrap_type(); - - auto scatter_update_key_m = wrap_type({past_param_key_m, indices_key_m, updates_key_m, axis_key_m}); - - auto vsplit_axis_key_m = any_input(); - auto vsplit_lengths_key_m = any_input(); - auto variadic_split_key_m = wrap_type( - {scatter_update_key_m, vsplit_axis_key_m, vsplit_lengths_key_m}); - - auto scatter_or_vsplit_key_m = std::make_shared( - OutputVector{scatter_update_key_m, variadic_split_key_m}); - - auto reshape1_pattern_key_m = any_input(); - auto reshape_4d_to_5d_key_m = wrap_type({scatter_or_vsplit_key_m, reshape1_pattern_key_m}); - - auto concat_key_m = wrap_type({reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m}); - - auto reshape2_pattern_key_m = any_input(); - auto reshape_5d_to_4d_key_m = wrap_type({concat_key_m, reshape2_pattern_key_m}); - - auto past_param_value_m = wrap_type(); - auto indices_value_m = any_input(); - auto updates_value_m = any_input(); - auto axis_value_m = wrap_type(); - - auto scatter_update_value_m = wrap_type({past_param_value_m, indices_value_m, updates_value_m, axis_value_m}); - - auto vsplit_axis_value_m = any_input(); - auto vsplit_lengths_value_m = any_input(); - auto variadic_split_value_m = wrap_type( - { scatter_update_value_m, vsplit_axis_value_m, vsplit_lengths_value_m }); - - auto scatter_or_vsplit_value_m = std::make_shared( - OutputVector{ scatter_update_value_m, variadic_split_value_m }); - - auto reshape1_pattern_value_m = any_input(); - auto reshape_4d_to_5d_value_m = wrap_type({scatter_or_vsplit_value_m, reshape1_pattern_value_m}); - - auto concat_value_m = wrap_type({reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m}); - - auto reshape2_pattern_value_m = any_input(); - auto reshape_5d_to_4d_value_m = wrap_type({concat_value_m, reshape2_pattern_value_m}); - - auto sdpa_without_attn_mask_m = wrap_type({input_a_m, reshape_5d_to_4d_key_m, reshape_5d_to_4d_value_m}); - auto sdpa_with_attn_mask_m = - wrap_type({input_a_m, reshape_5d_to_4d_key_m, reshape_5d_to_4d_value_m, any_input()}); - auto sdpa_with_attn_mask_and_scale_m = - wrap_type({input_a_m, reshape_5d_to_4d_key_m, reshape_5d_to_4d_value_m, input_attn_mask_m, input_scale_m}); - - - auto sdpa_m = std::make_shared(OutputVector{sdpa_without_attn_mask_m, sdpa_with_attn_mask_m, sdpa_with_attn_mask_and_scale_m}); - - ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](ov::pass::pattern::Matcher& m) { - const auto& pattern_map = m.get_pattern_value_map(); - - std::shared_ptr sdpa; - sdpa = std::dynamic_pointer_cast( - pattern_map.at(sdpa_m).get_node_shared_ptr()); - const auto past_param_key = - std::dynamic_pointer_cast(pattern_map.at(past_param_key_m).get_node_shared_ptr()); - const auto past_param_value = - std::dynamic_pointer_cast(pattern_map.at(past_param_value_m).get_node_shared_ptr()); - - - OutputVector data_inputs; - - if (pattern_map.count(scatter_update_key_m) > 0) { - const auto scatter_update_key = - std::dynamic_pointer_cast(pattern_map.at(scatter_update_key_m).get_node_shared_ptr()); - } if (pattern_map.count(variadic_split_key_m) > 0) { - const auto variadic_split_key = - std::dynamic_pointer_cast(pattern_map.at(variadic_split_key_m).get_node_shared_ptr()); - sdpa->input(1).replace_source_output(variadic_split_key->output(1)); - } - if (pattern_map.count(scatter_update_value_m) > 0) { - const auto scatter_update_value = - std::dynamic_pointer_cast(pattern_map.at(scatter_update_value_m).get_node_shared_ptr()); - } if (pattern_map.count(variadic_split_value_m) > 0) { - const auto variadic_split_value = - std::dynamic_pointer_cast(pattern_map.at(variadic_split_value_m).get_node_shared_ptr()); - sdpa->input(2).replace_source_output(variadic_split_value->output(1)); - - } - return true; - }; - auto m = std::make_shared(sdpa_m, "ScatterUpdateReshapeConcatReshapeSDPAFusion"); - - this->register_matcher(m, callback); -} - -} // namespace ov::intel_gpu \ No newline at end of file diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 9ad5a2cba9cbf6..c2659e02807126 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -114,7 +114,7 @@ #include "plugin/transformations/transpose_fusion.hpp" #include "plugin/transformations/unsqueeze_broadcast_reshape_matmul_fusion.hpp" #include "plugin/transformations/unsqueeze_broadcast_reshape_sdpa_fusion.hpp" -#include "plugin/transformations/scatterupdate_reshape_concat_reshape_sdpa_fusion.hpp" +#include "plugin/transformations/GQA_decomposition_fusion.hpp" #include "plugin/transformations/disable_fp16_comp_rms.hpp" #include "plugin/transformations/swiglu_fusion_with_clamp.hpp" #include "plugin/transformations/disable_fp16_comp_cumsum_sin_gen.hpp" @@ -1610,7 +1610,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { manager.register_pass(); } manager.register_pass(); - manager.register_pass(); + manager.register_pass(); manager.register_pass(); From 5d3d40dfacf9f1565482de44afdbb6a5c83afe7e Mon Sep 17 00:00:00 2001 From: Beheshti Date: Fri, 10 Jul 2026 13:11:34 -0700 Subject: [PATCH 03/15] correct header name --- .../src/plugin/transformations/GQA_decomposition_fusion.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp index ddba56032a33b8..4a76656cbf7c38 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "GQA_Decompositionfusion_fusion.hpp" +#include "GQA_Decomposition_fusion.hpp" #include "openvino/pass/pattern/matcher.hpp" #include "openvino/pass/pattern/op/or.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" From 2a0395addec60015eb58033956e45b0238dfe007 Mon Sep 17 00:00:00 2001 From: Beheshti Date: Fri, 10 Jul 2026 13:18:30 -0700 Subject: [PATCH 04/15] change transformation name --- .../src/plugin/transformations/GQA_decomposition_fusion.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp index 4a76656cbf7c38..e095f8647e3bdf 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp @@ -73,7 +73,7 @@ namespace ov::intel_gpu { } return true; }; - auto m = std::make_shared(sdpa_m, "ScatterUpdateReshapeConcatReshapeSDPAFusion"); + auto m = std::make_shared(sdpa_m, "GQADecompositionfusion"); this->register_matcher(m, callback); } From f3283eca42b3b594c960b02819bac74d88821043 Mon Sep 17 00:00:00 2001 From: Beheshti Date: Fri, 10 Jul 2026 14:07:52 -0700 Subject: [PATCH 05/15] add missing header files --- .../src/plugin/transformations/GQA_decomposition_fusion.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp index e095f8647e3bdf..fa2638030d2963 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp @@ -9,6 +9,8 @@ #include "openvino/op/reshape.hpp" #include "openvino/op/concat.hpp" #include "openvino/core/partial_shape.hpp" +#include "intel_gpu/op/sdpa.hpp" +#include namespace ov::intel_gpu { GQADecompositionfusion::GQADecompositionfusion() { From 22e1e26672a6f9dee8f9f4b2ee99d593c45334f7 Mon Sep 17 00:00:00 2001 From: Beheshti Date: Fri, 10 Jul 2026 15:07:36 -0700 Subject: [PATCH 06/15] update header name --- .../src/plugin/transformations/GQA_decomposition_fusion.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp index fa2638030d2963..724dd51cb315c1 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "GQA_Decomposition_fusion.hpp" +#include "GQA_decomposition_fusion.hpp" #include "openvino/pass/pattern/matcher.hpp" #include "openvino/pass/pattern/op/or.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" From 9f9a7797a1dd9b06429bb769794e335a3222b6db Mon Sep 17 00:00:00 2001 From: Beheshti Date: Fri, 10 Jul 2026 17:57:48 -0700 Subject: [PATCH 07/15] Add test case for GQA_decomposition_fusion --- .../GQA_decomposition_fusion_test.cpp | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 src/plugins/intel_gpu/tests/unit/transformations/GQA_decomposition_fusion_test.cpp diff --git a/src/plugins/intel_gpu/tests/unit/transformations/GQA_decomposition_fusion_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/GQA_decomposition_fusion_test.cpp new file mode 100644 index 00000000000000..9108e276f1ae12 --- /dev/null +++ b/src/plugins/intel_gpu/tests/unit/transformations/GQA_decomposition_fusion_test.cpp @@ -0,0 +1,102 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "common_test_utils/ov_test_utils.hpp" + +#include "ov_ops/rotary_positional_embeddings.hpp" + +#include "openvino/core/model.hpp" +#include "openvino/pass/manager.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/reshape.hpp" +#include "openvino/op/concat.hpp" +#include "openvino/op/scatter_update.hpp" +#include "openvino/op/variadic_split.hpp" + +#include "intel_gpu/op/sdpa.hpp" + +#include "plugin/transformations/GQA_decomposition_fusion.hpp" + +#include + +using namespace testing; +using namespace ov::intel_gpu; + +namespace ov { +namespace test { +namespace intel_gpu { + +TEST_F(TransformationTestsF, GQADecompositionFusion1) { + std::vector in0_order = {0, 1, 2, 3}; + std::vector in1_order = {0, 1, 2, 3}; + std::vector in2_order = {0, 1, 2, 3}; + std::vector out_order = {0, 1, 2, 3}; + const bool is_causal = false; + { + + auto input_q = std::make_shared(ov::element::f16, ov::Shape{1, 4, 8, 16}); + auto input_k = std::make_shared(ov::element::f16, ov::Shape{1, 2, 8, 16}); + auto input_v = std::make_shared(ov::element::f16, ov::Shape{1, 2, 8, 16}); + + auto indices = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {0}); + auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {1}); + auto split_lengths = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 1}); + auto k_updates = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{1, 1, 8, 16}, {0.0f}); + auto v_updates = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{1, 1, 8, 16}, {0.0f}); + + auto scatter_k = std::make_shared(input_k, indices, k_updates, axis); + auto scatter_v = std::make_shared(input_v, indices, v_updates, axis); + auto split_k = std::make_shared(scatter_k, axis, split_lengths); + auto split_v = std::make_shared(scatter_v, axis, split_lengths); + auto split_k_out = split_k->output(0); + auto split_v_out = split_v->output(0); + + auto reshape_k_5d_pattern = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, {1, 1, 1, 8, 16}); + auto reshape_v_5d_pattern = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, {1, 1, 1, 8, 16}); + + auto reshape_k_5d = std::make_shared(split_k_out, reshape_k_5d_pattern, false); + auto reshape_v_5d = std::make_shared(split_v_out, reshape_v_5d_pattern, false); + + auto concat_k = std::make_shared(ov::OutputVector{reshape_k_5d, reshape_k_5d, reshape_k_5d, reshape_k_5d},2); + auto concat_v = std::make_shared(ov::OutputVector{reshape_v_5d, reshape_v_5d, reshape_v_5d, reshape_v_5d},2); + + auto reshape_k_4d_pattern = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 4 }, { 1, 4, 8, 16 }); + auto reshape_v_4d_pattern = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 4 }, { 1, 4, 8, 16 }); + + auto reshape_k_4d = std::make_shared(concat_k, reshape_k_4d_pattern, false); + auto reshape_v_4d = std::make_shared(concat_v, reshape_v_4d_pattern, false); + + auto inputs = ov::OutputVector{input_q, reshape_k_4d, reshape_v_4d}; + auto sdpa = std::make_shared(inputs, is_causal, in0_order, in1_order, in2_order, out_order); + + model = std::make_shared(ov::OutputVector{ sdpa }, ov::ParameterVector{input_q, input_k, input_v}); + manager.register_pass(); + } + { + auto input_q = std::make_shared(ov::element::f16, ov::Shape{1, 4, 8, 16}); + auto input_k = std::make_shared(ov::element::f16, ov::Shape{1, 2, 8, 16}); + auto input_v = std::make_shared(ov::element::f16, ov::Shape{1, 2, 8, 16}); + + auto indices = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {0}); + auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {1}); + auto split_lengths = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 1}); + auto k_updates = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{1, 1, 8, 16}, {0.0f}); + auto v_updates = ov::op::v0::Constant::create(ov::element::f16, ov::Shape{1, 1, 8, 16}, {0.0f}); + + auto scatter_k = std::make_shared(input_k, indices, k_updates, axis); + auto scatter_v = std::make_shared(input_v, indices, v_updates, axis); + auto split_k = std::make_shared(scatter_k, axis, split_lengths); + auto split_v = std::make_shared(scatter_v, axis, split_lengths); + + auto inputs = ov::OutputVector{ input_q, split_k->output(1), split_v->output(1) }; + auto sdpa = std::make_shared(inputs, is_causal, in0_order, in1_order, in2_order, out_order); + + model_ref = std::make_shared(ov::OutputVector{ sdpa }, ov::ParameterVector{ input_q, input_k, input_v }); + comparator.enable(FunctionsComparator::ATTRIBUTES); + } +} + +} // namespace intel_gpu +} // namespace test +} // namespace ov \ No newline at end of file From 63d37fa32cd16b6521d768237a2b4bd8ede91217 Mon Sep 17 00:00:00 2001 From: Beheshti Date: Fri, 10 Jul 2026 18:00:36 -0700 Subject: [PATCH 08/15] delete space --- .../unit/transformations/GQA_decomposition_fusion_test.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/plugins/intel_gpu/tests/unit/transformations/GQA_decomposition_fusion_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/GQA_decomposition_fusion_test.cpp index 9108e276f1ae12..1e53c4ed15a943 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/GQA_decomposition_fusion_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/GQA_decomposition_fusion_test.cpp @@ -3,9 +3,6 @@ // #include "common_test_utils/ov_test_utils.hpp" - -#include "ov_ops/rotary_positional_embeddings.hpp" - #include "openvino/core/model.hpp" #include "openvino/pass/manager.hpp" #include "openvino/op/constant.hpp" @@ -13,11 +10,8 @@ #include "openvino/op/concat.hpp" #include "openvino/op/scatter_update.hpp" #include "openvino/op/variadic_split.hpp" - #include "intel_gpu/op/sdpa.hpp" - #include "plugin/transformations/GQA_decomposition_fusion.hpp" - #include using namespace testing; @@ -34,7 +28,6 @@ TEST_F(TransformationTestsF, GQADecompositionFusion1) { std::vector out_order = {0, 1, 2, 3}; const bool is_causal = false; { - auto input_q = std::make_shared(ov::element::f16, ov::Shape{1, 4, 8, 16}); auto input_k = std::make_shared(ov::element::f16, ov::Shape{1, 2, 8, 16}); auto input_v = std::make_shared(ov::element::f16, ov::Shape{1, 2, 8, 16}); From 3bd0fad69aaae1d41248b40643ce43794ecca245 Mon Sep 17 00:00:00 2001 From: Beheshti Date: Tue, 14 Jul 2026 10:22:42 -0700 Subject: [PATCH 09/15] change transformation name --- ...ion_fusion.cpp => KVCache_GQA_Broadcast_Elimination.cpp} | 6 +++--- ...ion_fusion.hpp => KVCache_GQA_Broadcast_Elimination.hpp} | 6 +++--- .../intel_gpu/src/plugin/transformations_pipeline.cpp | 4 ++-- ...usion_test.cpp => KVCache_GQA_Broadcast_Elimination.cpp} | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) rename src/plugins/intel_gpu/src/plugin/transformations/{GQA_decomposition_fusion.cpp => KVCache_GQA_Broadcast_Elimination.cpp} (96%) rename src/plugins/intel_gpu/src/plugin/transformations/{GQA_decomposition_fusion.hpp => KVCache_GQA_Broadcast_Elimination.hpp} (55%) rename src/plugins/intel_gpu/tests/unit/transformations/{GQA_decomposition_fusion_test.cpp => KVCache_GQA_Broadcast_Elimination.cpp} (96%) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp b/src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.cpp similarity index 96% rename from src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp rename to src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.cpp index 724dd51cb315c1..3c203cf326163e 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "GQA_decomposition_fusion.hpp" +#include "KVCache_GQA_Broadcast_Elimination.hpp" #include "openvino/pass/pattern/matcher.hpp" #include "openvino/pass/pattern/op/or.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" @@ -13,7 +13,7 @@ #include namespace ov::intel_gpu { - GQADecompositionfusion::GQADecompositionfusion() { + KVCacheGQABroadcastElimination::KVCacheGQABroadcastElimination() { using namespace ov::pass::pattern; auto reshape_4d_to_5d = [](const ov::Output& output) { @@ -75,7 +75,7 @@ namespace ov::intel_gpu { } return true; }; - auto m = std::make_shared(sdpa_m, "GQADecompositionfusion"); + auto m = std::make_shared(sdpa_m, "KVCacheGQABroadcastElimination"); this->register_matcher(m, callback); } diff --git a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.hpp b/src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.hpp similarity index 55% rename from src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.hpp rename to src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.hpp index 8bf222a0cd95e1..ef598d326d8fcd 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/GQA_decomposition_fusion.hpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.hpp @@ -8,10 +8,10 @@ namespace ov::intel_gpu { -class GQADecompositionfusion : public ov::pass::MatcherPass { +class KVCacheGQABroadcastElimination : public ov::pass::MatcherPass { public: - OPENVINO_MATCHER_PASS_RTTI("GQADecompositionfusion"); - GQADecompositionfusion(); + OPENVINO_MATCHER_PASS_RTTI("KVCacheGQABroadcastElimination"); + KVCacheGQABroadcastElimination(); }; } // namespace ov::intel_gpu diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index c2659e02807126..75908e8011d8c9 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -114,7 +114,7 @@ #include "plugin/transformations/transpose_fusion.hpp" #include "plugin/transformations/unsqueeze_broadcast_reshape_matmul_fusion.hpp" #include "plugin/transformations/unsqueeze_broadcast_reshape_sdpa_fusion.hpp" -#include "plugin/transformations/GQA_decomposition_fusion.hpp" +#include "plugin/transformations/KVCache_GQA_Broadcast_Elimination.hpp" #include "plugin/transformations/disable_fp16_comp_rms.hpp" #include "plugin/transformations/swiglu_fusion_with_clamp.hpp" #include "plugin/transformations/disable_fp16_comp_cumsum_sin_gen.hpp" @@ -1610,7 +1610,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { manager.register_pass(); } manager.register_pass(); - manager.register_pass(); + manager.register_pass(); manager.register_pass(); diff --git a/src/plugins/intel_gpu/tests/unit/transformations/GQA_decomposition_fusion_test.cpp b/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp similarity index 96% rename from src/plugins/intel_gpu/tests/unit/transformations/GQA_decomposition_fusion_test.cpp rename to src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp index 1e53c4ed15a943..9bceb7c48932d3 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/GQA_decomposition_fusion_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp @@ -11,7 +11,7 @@ #include "openvino/op/scatter_update.hpp" #include "openvino/op/variadic_split.hpp" #include "intel_gpu/op/sdpa.hpp" -#include "plugin/transformations/GQA_decomposition_fusion.hpp" +#include "plugin/transformations/KVCache_GQA_Broadcast_Elimination.hpp" #include using namespace testing; @@ -21,7 +21,7 @@ namespace ov { namespace test { namespace intel_gpu { -TEST_F(TransformationTestsF, GQADecompositionFusion1) { +TEST_F(TransformationTestsF, KVCacheGQABroadcastEliminationTest1) { std::vector in0_order = {0, 1, 2, 3}; std::vector in1_order = {0, 1, 2, 3}; std::vector in2_order = {0, 1, 2, 3}; @@ -64,7 +64,7 @@ TEST_F(TransformationTestsF, GQADecompositionFusion1) { auto sdpa = std::make_shared(inputs, is_causal, in0_order, in1_order, in2_order, out_order); model = std::make_shared(ov::OutputVector{ sdpa }, ov::ParameterVector{input_q, input_k, input_v}); - manager.register_pass(); + manager.register_pass(); } { auto input_q = std::make_shared(ov::element::f16, ov::Shape{1, 4, 8, 16}); From 7cdd7df7dfecf2cbc4373e0502d361b291bb3970 Mon Sep 17 00:00:00 2001 From: Beheshti Date: Tue, 14 Jul 2026 12:35:48 -0700 Subject: [PATCH 10/15] apply reviewer comments --- .../KVCache_GQA_Broadcast_Elimination.cpp | 9 ++++----- .../intel_gpu/src/plugin/transformations_pipeline.cpp | 2 +- .../KVCache_GQA_Broadcast_Elimination.cpp | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.cpp b/src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.cpp index 3c203cf326163e..07ec101603c52e 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.cpp @@ -43,12 +43,12 @@ namespace ov::intel_gpu { auto in_value = any_input(); auto reshape1_pattern_value_m = any_input(); - auto reshape_4d_to_5d_value_m = wrap_type({ in_value, reshape1_pattern_value_m }); + auto reshape_4d_to_5d_value_m = wrap_type({ in_value, reshape1_pattern_value_m }, reshape_4d_to_5d); auto concat_value_m = wrap_type({ reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m }); auto reshape2_pattern_value_m = any_input(); - auto reshape_5d_to_4d_value_m = wrap_type({ concat_value_m, reshape2_pattern_value_m }); + auto reshape_5d_to_4d_value_m = wrap_type({ concat_value_m, reshape2_pattern_value_m }, reshape_5d_to_4d); auto sdpa_without_attn_mask_m = wrap_type({ input_a_m, reshape_5d_to_4d_key_m, reshape_5d_to_4d_value_m }); auto sdpa_with_attn_mask_m = @@ -71,13 +71,12 @@ namespace ov::intel_gpu { if (pattern_map.count(in_value) > 0) { const auto value = pattern_map.at(in_value).get_node_shared_ptr(); sdpa->input(2).replace_source_output(value->output(1)); - } - return true; + return false; }; auto m = std::make_shared(sdpa_m, "KVCacheGQABroadcastElimination"); this->register_matcher(m, callback); } -} // namespace ov::intel_gpu \ No newline at end of file +} // namespace ov::intel_gpu diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 75908e8011d8c9..47576230b9c72b 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -114,7 +114,7 @@ #include "plugin/transformations/transpose_fusion.hpp" #include "plugin/transformations/unsqueeze_broadcast_reshape_matmul_fusion.hpp" #include "plugin/transformations/unsqueeze_broadcast_reshape_sdpa_fusion.hpp" -#include "plugin/transformations/KVCache_GQA_Broadcast_Elimination.hpp" +#include "plugin/transformations/kvcache_gqa_broadcast_elimination.hpp" #include "plugin/transformations/disable_fp16_comp_rms.hpp" #include "plugin/transformations/swiglu_fusion_with_clamp.hpp" #include "plugin/transformations/disable_fp16_comp_cumsum_sin_gen.hpp" diff --git a/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp b/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp index 9bceb7c48932d3..030d905bec2bd8 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp @@ -11,7 +11,7 @@ #include "openvino/op/scatter_update.hpp" #include "openvino/op/variadic_split.hpp" #include "intel_gpu/op/sdpa.hpp" -#include "plugin/transformations/KVCache_GQA_Broadcast_Elimination.hpp" +#include "plugin/transformations/kvcache_gqa_broadcast_elimination.hpp" #include using namespace testing; From 54043f8c63497f2a8b62c3079ca5c2cde6194d7b Mon Sep 17 00:00:00 2001 From: Beheshti Date: Tue, 14 Jul 2026 12:42:45 -0700 Subject: [PATCH 11/15] file name change (capital to small) --- ...cast_Elimination.cpp => kvcache_gqa_broadcast_elimination.cpp} | 0 ...cast_Elimination.hpp => kvcache_gqa_broadcast_elimination.hpp} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/plugins/intel_gpu/src/plugin/transformations/{KVCache_GQA_Broadcast_Elimination.cpp => kvcache_gqa_broadcast_elimination.cpp} (100%) rename src/plugins/intel_gpu/src/plugin/transformations/{KVCache_GQA_Broadcast_Elimination.hpp => kvcache_gqa_broadcast_elimination.hpp} (100%) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.cpp b/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp similarity index 100% rename from src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.cpp rename to src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp diff --git a/src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.hpp b/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.hpp similarity index 100% rename from src/plugins/intel_gpu/src/plugin/transformations/KVCache_GQA_Broadcast_Elimination.hpp rename to src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.hpp From 6aba2cb04a0ed7cfcfbe7b9af6770d5bed5a3a7a Mon Sep 17 00:00:00 2001 From: Beheshti Date: Tue, 14 Jul 2026 12:55:43 -0700 Subject: [PATCH 12/15] update header file name --- .../transformations/kvcache_gqa_broadcast_elimination.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp b/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp index 07ec101603c52e..87950a04d804bc 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "KVCache_GQA_Broadcast_Elimination.hpp" +#include "kvcache_gqa_broadcast_elimination.hpp" #include "openvino/pass/pattern/matcher.hpp" #include "openvino/pass/pattern/op/or.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" From e3d5b1f35a3a6c73469030db8223eb9f7a0f25d9 Mon Sep 17 00:00:00 2001 From: Beheshti Date: Tue, 14 Jul 2026 15:25:48 -0700 Subject: [PATCH 13/15] apply reviwer comments --- .../transformations/kvcache_gqa_broadcast_elimination.cpp | 6 ++---- .../transformations/KVCache_GQA_Broadcast_Elimination.cpp | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp b/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp index 87950a04d804bc..24caad9c241de2 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp @@ -64,13 +64,11 @@ namespace ov::intel_gpu { auto sdpa = ov::as_type_ptr(m.get_match_root()); if (pattern_map.count(in_key) > 0) { - const auto key = pattern_map.at(in_key).get_node_shared_ptr(); - sdpa->input(1).replace_source_output(key->output(1)); + sdpa->input(1).replace_source_output(pattern_map.at(in_key)); } if (pattern_map.count(in_value) > 0) { - const auto value = pattern_map.at(in_value).get_node_shared_ptr(); - sdpa->input(2).replace_source_output(value->output(1)); + sdpa->input(2).replace_source_output(pattern_map.at(in_value)); } return false; }; diff --git a/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp b/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp index 030d905bec2bd8..d5377f6f279a4f 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp @@ -82,7 +82,7 @@ TEST_F(TransformationTestsF, KVCacheGQABroadcastEliminationTest1) { auto split_k = std::make_shared(scatter_k, axis, split_lengths); auto split_v = std::make_shared(scatter_v, axis, split_lengths); - auto inputs = ov::OutputVector{ input_q, split_k->output(1), split_v->output(1) }; + auto inputs = ov::OutputVector{ input_q, split_k->output(0), split_v->output(0) }; auto sdpa = std::make_shared(inputs, is_causal, in0_order, in1_order, in2_order, out_order); model_ref = std::make_shared(ov::OutputVector{ sdpa }, ov::ParameterVector{ input_q, input_k, input_v }); From 71db8e49df7ea160699f1191a8fa4f579c4b7bbd Mon Sep 17 00:00:00 2001 From: Beheshti Date: Wed, 15 Jul 2026 07:54:39 -0700 Subject: [PATCH 14/15] apply reviwer comments --- .../kvcache_gqa_broadcast_elimination.cpp | 13 ++++++------- ...p => kvcache_gqa_broadcast_elimination_test.cpp} | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) rename src/plugins/intel_gpu/tests/unit/transformations/{KVCache_GQA_Broadcast_Elimination.cpp => kvcache_gqa_broadcast_elimination_test.cpp} (99%) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp b/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp index 24caad9c241de2..5b2d31f50805a3 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp @@ -63,15 +63,14 @@ namespace ov::intel_gpu { auto sdpa = ov::as_type_ptr(m.get_match_root()); - if (pattern_map.count(in_key) > 0) { - sdpa->input(1).replace_source_output(pattern_map.at(in_key)); + if (pattern_map.count(in_key) == 0 || pattern_map.count(in_value) == 0) { + return false; } + sdpa->input(1).replace_source_output(pattern_map.at(in_key)); + sdpa->input(2).replace_source_output(pattern_map.at(in_value)); - if (pattern_map.count(in_value) > 0) { - sdpa->input(2).replace_source_output(pattern_map.at(in_value)); - } - return false; - }; + return true; + }; auto m = std::make_shared(sdpa_m, "KVCacheGQABroadcastElimination"); this->register_matcher(m, callback); diff --git a/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp b/src/plugins/intel_gpu/tests/unit/transformations/kvcache_gqa_broadcast_elimination_test.cpp similarity index 99% rename from src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp rename to src/plugins/intel_gpu/tests/unit/transformations/kvcache_gqa_broadcast_elimination_test.cpp index d5377f6f279a4f..183a57a4ffd66d 100644 --- a/src/plugins/intel_gpu/tests/unit/transformations/KVCache_GQA_Broadcast_Elimination.cpp +++ b/src/plugins/intel_gpu/tests/unit/transformations/kvcache_gqa_broadcast_elimination_test.cpp @@ -92,4 +92,4 @@ TEST_F(TransformationTestsF, KVCacheGQABroadcastEliminationTest1) { } // namespace intel_gpu } // namespace test -} // namespace ov \ No newline at end of file +} // namespace ov From df811adf99bcf00d46759e24f6f06664b747d009 Mon Sep 17 00:00:00 2001 From: Beheshti Date: Thu, 16 Jul 2026 16:04:06 -0700 Subject: [PATCH 15/15] change pattern matching --- .../kvcache_gqa_broadcast_elimination.cpp | 54 +++++++++++-------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp b/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp index 5b2d31f50805a3..1061e0a6a6c256 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/kvcache_gqa_broadcast_elimination.cpp @@ -15,10 +15,12 @@ namespace ov::intel_gpu { KVCacheGQABroadcastElimination::KVCacheGQABroadcastElimination() { using namespace ov::pass::pattern; - - auto reshape_4d_to_5d = [](const ov::Output& output) { - auto in_ps = output.get_node()->get_input_partial_shape(0); - auto out_ps = output.get_node()->get_output_partial_shape(0); + auto is_reshape_4d_to_5d = [](const ov::Node* node) { + if (!ov::is_type(node)) { + return false; + } + auto in_ps = node->get_input_partial_shape(0); + auto out_ps = node->get_output_partial_shape(0); return in_ps.rank().is_static() && out_ps.rank().is_static() && in_ps.size() == 4 && out_ps.size() == 5; }; @@ -28,24 +30,34 @@ namespace ov::intel_gpu { return in_ps.rank().is_static() && out_ps.rank().is_static() && in_ps.size() == 5 && out_ps.size() == 4; }; + auto broadcast_concat = [is_reshape_4d_to_5d](const ov::Output& output) { + auto concat = ov::as_type(output.get_node()); + if (concat == nullptr || concat->get_input_size() < 2) { + return false; + } + auto first = concat->get_input_node_ptr(0); + if (!is_reshape_4d_to_5d(first)) { + return false; + } + for (size_t i = 1; i < concat->get_input_size(); ++i) { + if (concat->get_input_node_ptr(i) != first) { + return false; + + } + } + return true; + }; + auto input_a_m = any_input(); auto input_attn_mask_m = any_input(); auto input_scale_m = any_input(); - auto in_key = any_input(); - auto reshape1_pattern_key_m = any_input(); - auto reshape_4d_to_5d_key_m = wrap_type({ in_key, reshape1_pattern_key_m }, reshape_4d_to_5d); - - auto concat_key_m = wrap_type({ reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m, reshape_4d_to_5d_key_m }); + auto concat_key_m = wrap_type(broadcast_concat); auto reshape2_pattern_key_m = any_input(); auto reshape_5d_to_4d_key_m = wrap_type({ concat_key_m, reshape2_pattern_key_m }, reshape_5d_to_4d); - auto in_value = any_input(); - auto reshape1_pattern_value_m = any_input(); - auto reshape_4d_to_5d_value_m = wrap_type({ in_value, reshape1_pattern_value_m }, reshape_4d_to_5d); - - auto concat_value_m = wrap_type({ reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m, reshape_4d_to_5d_value_m }); + auto concat_value_m = wrap_type(broadcast_concat); auto reshape2_pattern_value_m = any_input(); auto reshape_5d_to_4d_value_m = wrap_type({ concat_value_m, reshape2_pattern_value_m }, reshape_5d_to_4d); @@ -60,19 +72,19 @@ namespace ov::intel_gpu { ov::matcher_pass_callback callback = [OV_CAPTURE_CPY_AND_THIS](ov::pass::pattern::Matcher& m) { const auto& pattern_map = m.get_pattern_value_map(); - auto sdpa = ov::as_type_ptr(m.get_match_root()); - - if (pattern_map.count(in_key) == 0 || pattern_map.count(in_value) == 0) { + if (pattern_map.count(concat_key_m) == 0 || pattern_map.count(concat_value_m) == 0) { return false; } - sdpa->input(1).replace_source_output(pattern_map.at(in_key)); - sdpa->input(2).replace_source_output(pattern_map.at(in_value)); - + // Concat -> Reshape(4d->5d) -> original source that was broadcast. + auto broadcast_source = [](const ov::Output& concat_out) { + return concat_out.get_node()->input_value(0).get_node()->input_value(0); + }; + sdpa->input(1).replace_source_output(broadcast_source(pattern_map.at(concat_key_m))); + sdpa->input(2).replace_source_output(broadcast_source(pattern_map.at(concat_value_m))); return true; }; auto m = std::make_shared(sdpa_m, "KVCacheGQABroadcastElimination"); - this->register_matcher(m, callback); }