From 16655b29f38f3ab41fdc6c0bd32c0e1d60cb3b4f Mon Sep 17 00:00:00 2001 From: wilson-seok Date: Wed, 15 Jul 2026 13:44:36 +0900 Subject: [PATCH 1/2] [GPU] Fix fused eltwise rank reduction Keep per-channel eltwise transformations correct across rank-changing data movement. Canonicalize provable higher-rank fused peers against the actual reduced layout and preserve producer rank when no exact fold exists. Add regression coverage for 5D and exhaustive 6D broadcast masks. --- .../graph/impls/ocl/kernel_selector_helper.h | 25 +- .../src/graph/include/fused_shape_utils.hpp | 71 ++ .../intel_gpu/src/graph/layout_optimizer.cpp | 43 +- ...e_eltwise_up_data_movement_per_channel.cpp | 178 ++++ ...e_eltwise_up_data_movement_per_channel.hpp | 17 + .../src/plugin/transformations_pipeline.cpp | 3 +- .../unit/fusions/permute_fusion_test.cpp | 764 ++++++++++++++++++ .../canonicalize_fused_shapes_test.cpp | 161 ++++ ...wise_up_data_movement_per_channel_test.cpp | 234 ++++++ 9 files changed, 1491 insertions(+), 5 deletions(-) create mode 100644 src/plugins/intel_gpu/src/graph/include/fused_shape_utils.hpp create mode 100644 src/plugins/intel_gpu/src/plugin/transformations/move_eltwise_up_data_movement_per_channel.cpp create mode 100644 src/plugins/intel_gpu/src/plugin/transformations/move_eltwise_up_data_movement_per_channel.hpp create mode 100644 src/plugins/intel_gpu/tests/unit/module_tests/canonicalize_fused_shapes_test.cpp create mode 100644 src/plugins/intel_gpu/tests/unit/transformations/move_eltwise_up_data_movement_per_channel_test.cpp diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h index b7c5883fc1c5..5347ca0d6993 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.h @@ -17,6 +17,10 @@ #include "intel_gpu/primitives/reorder.hpp" #include "intel_gpu/primitives/primitive.hpp" +#include "fused_shape_utils.hpp" + +#include "openvino/core/shape.hpp" + #include "kernel_selector_params.h" #include "weight_bias_params.h" #include "kernel_selector_common.h" @@ -269,13 +273,30 @@ inline kernel_impl_params canonicalize_fused_shapes(const kernel_impl_params& im for (auto& fd : updated_impl_params.fused_desc) { if (fd.is_type() && fd.total_num_deps == 2 && fd.has_outer_dep()) { if (updated_impl_params.input_layouts.size() > size_t(fd.outer_dep_start_idx)) { - const auto& out_pshape = updated_impl_params.output_layouts[0].get_partial_shape(); + const auto& out_layout = updated_impl_params.output_layouts[0]; + const auto& out_pshape = out_layout.get_partial_shape(); auto& dep_layout = updated_impl_params.input_layouts[fd.outer_dep_start_idx]; const auto& dep_shape = dep_layout.get_partial_shape(); if (!broadcastable(dep_shape, out_pshape, use_new_shape_infer)) { - dep_layout.set_partial_shape(extend_shape_to_rank_from_begin(dep_shape, out_pshape.size())); + if (dep_shape.size() > out_pshape.size()) { + // extend_shape_to_rank_from_begin() can only prepend unit dims; it leaves a + // higher-rank peer unchanged, so the fused-op kernel would read the peer with a + // rank/format inconsistent with the host iteration space (df1 output-0 defect). + // Fold the peer onto the host rank when it is a provably order-preserving reshape + // (equal-total planar reshape or a legal planar broadcast); otherwise leave it to + // the pre-existing rank-extension path. Optimization inapplicability is not an + // error, so no assertion is raised and valid models always compile. + if (auto folded = fold_higher_rank_fused_peer(dep_layout, out_layout)) { + dep_layout.set_partial_shape(*folded); + dep_layout.format = format::adjust_to_rank(dep_layout.format, out_pshape.size()); + } else { + dep_layout.set_partial_shape(extend_shape_to_rank_from_begin(dep_shape, out_pshape.size())); + } + } else { + dep_layout.set_partial_shape(extend_shape_to_rank_from_begin(dep_shape, out_pshape.size())); + } } } } diff --git a/src/plugins/intel_gpu/src/graph/include/fused_shape_utils.hpp b/src/plugins/intel_gpu/src/graph/include/fused_shape_utils.hpp new file mode 100644 index 000000000000..a78736fa980f --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/include/fused_shape_utils.hpp @@ -0,0 +1,71 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "intel_gpu/runtime/layout.hpp" +#include "openvino/core/shape_util.hpp" + +namespace cldnn { + +// Computes a lower-rank representation of a fused eltwise peer when collapsing its leading spatial +// axes is an order-preserving reshape and the result broadcasts directly to the host layout. +inline std::optional fold_higher_rank_fused_peer(const layout& peer_layout, const layout& host_layout) { + const auto& peer_shape = peer_layout.get_partial_shape(); + const auto& host_shape = host_layout.get_partial_shape(); + + const size_t peer_rank = peer_shape.size(); + const size_t host_rank = host_shape.size(); + if (peer_rank <= host_rank || host_rank < 3) + return std::nullopt; + if (peer_shape.is_dynamic() || host_shape.is_dynamic()) + return std::nullopt; + if (peer_layout.data_padding || host_layout.data_padding) + return std::nullopt; + + const auto& peer_format = peer_layout.format; + const auto& host_format = host_layout.format; + if (!format::is_simple_data_format(peer_format) || !format::is_simple_data_format(host_format)) + return std::nullopt; + if (!format::is_default_format(peer_format) || !format::is_default_format(host_format)) + return std::nullopt; + if (format::adjust_to_rank(peer_format, host_rank) != host_format) + return std::nullopt; + + const auto peer_dims = peer_shape.to_shape(); + const auto host_dims = host_shape.to_shape(); + const size_t fold_count = peer_rank - host_rank + 1; + ov::Shape folded_dims; + folded_dims.reserve(host_rank); + folded_dims.push_back(peer_dims[0]); + folded_dims.push_back(peer_dims[1]); + + size_t grouped = 1; + for (size_t i = 2; i < 2 + fold_count; ++i) { + const auto grouped_size = ov::util::shape_size_safe({grouped, peer_dims[i]}); + if (!grouped_size.has_value()) + return std::nullopt; + grouped = grouped_size.value(); + } + folded_dims.push_back(grouped); + folded_dims.insert(folded_dims.end(), peer_dims.begin() + 2 + fold_count, peer_dims.end()); + + const auto peer_total = ov::util::shape_size_safe(peer_dims); + const auto folded_total = ov::util::shape_size_safe(folded_dims); + if (!peer_total.has_value() || !folded_total.has_value() || peer_total.value() != folded_total.value()) + return std::nullopt; + if (folded_dims.size() != host_dims.size()) + return std::nullopt; + + for (size_t i = 0; i < folded_dims.size(); ++i) { + if (folded_dims[i] != 1 && folded_dims[i] != host_dims[i]) + return std::nullopt; + } + + return ov::PartialShape(folded_dims); +} + +} // namespace cldnn \ No newline at end of file diff --git a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp index 766fad3be4b8..364e65d3bc02 100644 --- a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp +++ b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp @@ -47,6 +47,7 @@ #include "lstm_seq_inst.h" #include "group_normalization_inst.h" #include "to_string_utils.h" +#include "fused_shape_utils.hpp" #include #include #include @@ -69,6 +70,37 @@ static size_t get_post_ops_count(const program_node& node) { return onednn_post_ops_count; } +// A rank-reducing reorder may be fused into a producer only when every higher-rank external eltwise +// peer has the same provable lower-rank representation that OCL fused-op canonicalization will use. +static bool fused_peers_can_fold_to_layout(const program_node& prev, const layout& reduced_layout) { + if (!prev.has_fused_primitives()) + return true; + + const size_t reduced_rank = reduced_layout.get_rank(); + if (prev.get_output_layout().get_rank() <= reduced_rank) + return true; + + for (const auto& fd : prev.get_fused_primitives()) { + if (!fd.is_type() || !fd.has_outer_dep()) + continue; + + const auto outer_idx = static_cast(fd.outer_dep_start_idx); + const size_t outer_count = fd.deps.size(); + if (outer_idx >= prev.get_dependencies().size() || outer_count == 0 || outer_count > prev.get_dependencies().size() - outer_idx) { + return false; + } + + for (size_t i = 0; i < outer_count; ++i) { + const auto peer_layout = prev.get_dependency(outer_idx + i).get_output_layout(); + if (peer_layout.get_rank() > reduced_rank && !fold_higher_rank_fused_peer(peer_layout, reduced_layout).has_value()) { + return false; + } + } + } + + return true; +} + std::pair, bool> reorder_factory::get_reorder(primitive_id src_id, int32_t src_port, const layout& in_layout, @@ -391,6 +423,14 @@ bool layout_optimizer::can_fuse_reorder(program_node& prev, program_node& next, bool layout_optimizer::can_fuse_reorder_to_prev(program_node& prev, reorder_node& node, format fmt_prev, format fmt_next) { bool allow_new_shape_infer = node.get_program().is_new_shape_infer(); + + // Do not fuse a rank-reducing reorder into a producer whose fused higher-rank + // eltwise peer cannot be indexed correctly at the reduced rank (inner-spatial + // broadcast over the collapsed axes). Keeping the producer at its higher rank + // keeps the fused peer rank-consistent so the fused-op kernel reads it right. + if (!node.get_output_layout().is_dynamic() && !fused_peers_can_fold_to_layout(prev, node.get_output_layout())) { + return false; + } // Because kernels can work cross-layout, if reorder only performs type conversion, // fusing reorder to the previous node can be done even if it is a dynamic shape case if ((format::is_simple_data_format(fmt_prev) && format::is_simple_data_format(fmt_next)) && @@ -398,8 +438,7 @@ bool layout_optimizer::can_fuse_reorder_to_prev(program_node& prev, reorder_node // We can void only that case if we can check whether the current node is backedge of the network. // However no such handle is existing yet. (To be done in the future when we need to optimize out the type converting // reorders in the body network) - !node.get_program().is_body_program() && !prev.is_in_shape_of_subgraph() && - prev.get_preferred_impl_type() != cldnn::impl_types::cpu) { + !node.get_program().is_body_program() && !prev.is_in_shape_of_subgraph() && prev.get_preferred_impl_type() != cldnn::impl_types::cpu) { // case for truncate mode if ((prev.is_type() || prev.is_type() || prev.is_type() || prev.is_type() || prev.is_type