Skip to content

Commit 0cd964a

Browse files
intbfintbf
authored
[GPU] Block the fc horizontal fusion for weights that are not constants (#36572)
### Details: Found when using weight sharing with llms. In that case weights are converted into inputs/parameters, and the fusion might not work as expected. With inputs/parameters the constant folding won't happen, so the fusion will be invoked on every inference. Plus in some cases the concat operator might not be enabled for all weight types (like int4) ### Tickets: CVS-189072 ### AI Assistance: - AI assistance used: yes - do the code changes + unit tests
1 parent c508c75 commit 0cd964a

2 files changed

Lines changed: 134 additions & 1 deletion

File tree

‎src/plugins/intel_gpu/src/plugin/transformations/fc_horizontal_fusion.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,15 @@ FullyConnectedHorizontalFusion::FullyConnectedHorizontalFusion(bool fuse_mlp_swi
7070
const auto& fc_user = ov::as_type_ptr<op::FullyConnectedCompressed>(u);
7171
if (!fc_user)
7272
continue;
73+
74+
// Skip horizontal fusion when the weight is not a constant. The fused-weight Concat
75+
// created during fusion relies on constant-folding at compile time; with a non-constant
76+
// weight (e.g. weights provided as runtime inputs) it cannot fold, survives to program
77+
// build, and may hit formats/types without a Concat implementation. Even when a Concat
78+
// impl exists, concatenating weights on every inference is pure overhead with no benefit.
79+
if (!is_constant(fc_user->get_input_node_shared_ptr(1)))
80+
return false;
81+
7382
auto num_inputs = fc_user->inputs().size();
7483
if (num_inputs >= 5)
7584
nodes_with_zp++;

‎src/plugins/intel_gpu/tests/unit/transformations/horizontal_fc_fusion_test.cpp‎

Lines changed: 125 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "openvino/op/reshape.hpp"
2020
#include "openvino/op/add.hpp"
2121
#include "openvino/op/multiply.hpp"
22+
#include "openvino/op/convert.hpp"
2223
#include "openvino/pass/manager.hpp"
2324

2425
#include <transformations/utils/utils.hpp>
@@ -96,7 +97,130 @@ TEST_F(TransformationTestsF, FullyConnectedHorizontalFusion_no_bias_no_zp) {
9697
}
9798
}
9899

99-
TEST_F(TransformationTestsF, FullyConnectedHorizontalFusion_bias_zp) {
100+
TEST_F(TransformationTestsF, FullyConnectedHorizontalFusion_parameter_weights_no_fusion) {
101+
// Weights-as-inputs / share_weights rewrites weight Constants into Parameters. The fused-weight
102+
// Concat would be unfoldable and reach GPU program build (no i4/u4 concat kernel). Horizontal
103+
// fusion must NOT fire in this case, so model_ref is identical to model (3 separate FCs).
104+
std::vector<int64_t> pattern = {7, -1};
105+
{
106+
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{-1, 7, 4096});
107+
auto weight1 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{1024, 4096});
108+
weight1->set_friendly_name("weight1_1");
109+
auto weight2 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{512, 4096});
110+
weight2->set_friendly_name("weight1_2");
111+
auto weight3 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{128, 4096});
112+
weight3->set_friendly_name("weight1_3");
113+
auto bias1 = std::make_shared<ov::intel_gpu::op::Placeholder>();
114+
auto bias2 = std::make_shared<ov::intel_gpu::op::Placeholder>();
115+
auto bias3 = std::make_shared<ov::intel_gpu::op::Placeholder>();
116+
auto scale1 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{1024, 32});
117+
auto scale2 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{512, 32});
118+
auto scale3 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{128, 32});
119+
auto fc1 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight1, bias1, scale1);
120+
fc1->set_friendly_name("fc1");
121+
auto fc2 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight2, bias2, scale2);
122+
auto fc3 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight3, bias3, scale3);
123+
auto reshape_pattern = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{2}, pattern);
124+
auto reshape1 = std::make_shared<ov::op::v1::Reshape>(fc1, reshape_pattern, true);
125+
auto reshape2 = std::make_shared<ov::op::v1::Reshape>(fc2, reshape_pattern, true);
126+
auto reshape3 = std::make_shared<ov::op::v1::Reshape>(fc3, reshape_pattern, true);
127+
auto result1 = std::make_shared<ov::op::v0::Result>(reshape1);
128+
auto result2 = std::make_shared<ov::op::v0::Result>(reshape2);
129+
auto result3 = std::make_shared<ov::op::v0::Result>(reshape3);
130+
model = std::make_shared<ov::Model>(ov::ResultVector{result1, result2, result3}, ov::ParameterVector{input, weight1, weight2, weight3});
131+
manager.register_pass<FullyConnectedHorizontalFusion>();
132+
}
133+
{
134+
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{-1, 7, 4096});
135+
auto weight1 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{1024, 4096});
136+
weight1->set_friendly_name("weight1_1");
137+
auto weight2 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{512, 4096});
138+
weight2->set_friendly_name("weight1_2");
139+
auto weight3 = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{128, 4096});
140+
weight3->set_friendly_name("weight1_3");
141+
auto bias1 = std::make_shared<ov::intel_gpu::op::Placeholder>();
142+
auto bias2 = std::make_shared<ov::intel_gpu::op::Placeholder>();
143+
auto bias3 = std::make_shared<ov::intel_gpu::op::Placeholder>();
144+
auto scale1 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{1024, 32});
145+
auto scale2 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{512, 32});
146+
auto scale3 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{128, 32});
147+
auto fc1 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight1, bias1, scale1);
148+
fc1->set_friendly_name("fc1");
149+
auto fc2 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight2, bias2, scale2);
150+
auto fc3 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight3, bias3, scale3);
151+
auto reshape_pattern = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{2}, pattern);
152+
auto reshape1 = std::make_shared<ov::op::v1::Reshape>(fc1, reshape_pattern, true);
153+
auto reshape2 = std::make_shared<ov::op::v1::Reshape>(fc2, reshape_pattern, true);
154+
auto reshape3 = std::make_shared<ov::op::v1::Reshape>(fc3, reshape_pattern, true);
155+
auto result1 = std::make_shared<ov::op::v0::Result>(reshape1);
156+
auto result2 = std::make_shared<ov::op::v0::Result>(reshape2);
157+
auto result3 = std::make_shared<ov::op::v0::Result>(reshape3);
158+
model_ref = std::make_shared<ov::Model>(ov::ResultVector{result1, result2, result3}, ov::ParameterVector{input, weight1, weight2, weight3});
159+
comparator.enable(FunctionsComparator::ATTRIBUTES);
160+
}
161+
}
162+
163+
TEST_F(TransformationTestsF, FullyConnectedHorizontalFusion_convert_parameter_weights_no_fusion) {
164+
// Convert(Parameter) weight form must also block horizontal fusion (is_constant returns false
165+
// because the Convert input is a Parameter, not a Constant).
166+
std::vector<int64_t> pattern = {7, -1};
167+
{
168+
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{-1, 7, 4096});
169+
auto weight1_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{1024, 4096});
170+
auto weight2_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{512, 4096});
171+
auto weight3_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{128, 4096});
172+
auto weight1 = std::make_shared<ov::op::v0::Convert>(weight1_p, ov::element::f16);
173+
auto weight2 = std::make_shared<ov::op::v0::Convert>(weight2_p, ov::element::f16);
174+
auto weight3 = std::make_shared<ov::op::v0::Convert>(weight3_p, ov::element::f16);
175+
auto bias1 = std::make_shared<ov::intel_gpu::op::Placeholder>();
176+
auto bias2 = std::make_shared<ov::intel_gpu::op::Placeholder>();
177+
auto bias3 = std::make_shared<ov::intel_gpu::op::Placeholder>();
178+
auto scale1 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{1024, 32});
179+
auto scale2 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{512, 32});
180+
auto scale3 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{128, 32});
181+
auto fc1 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight1, bias1, scale1);
182+
auto fc2 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight2, bias2, scale2);
183+
auto fc3 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight3, bias3, scale3);
184+
auto reshape_pattern = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{2}, pattern);
185+
auto reshape1 = std::make_shared<ov::op::v1::Reshape>(fc1, reshape_pattern, true);
186+
auto reshape2 = std::make_shared<ov::op::v1::Reshape>(fc2, reshape_pattern, true);
187+
auto reshape3 = std::make_shared<ov::op::v1::Reshape>(fc3, reshape_pattern, true);
188+
auto result1 = std::make_shared<ov::op::v0::Result>(reshape1);
189+
auto result2 = std::make_shared<ov::op::v0::Result>(reshape2);
190+
auto result3 = std::make_shared<ov::op::v0::Result>(reshape3);
191+
model = std::make_shared<ov::Model>(ov::ResultVector{result1, result2, result3}, ov::ParameterVector{input, weight1_p, weight2_p, weight3_p});
192+
manager.register_pass<FullyConnectedHorizontalFusion>();
193+
}
194+
{
195+
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{-1, 7, 4096});
196+
auto weight1_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{1024, 4096});
197+
auto weight2_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{512, 4096});
198+
auto weight3_p = std::make_shared<ov::op::v0::Parameter>(ov::element::u4, ov::Shape{128, 4096});
199+
auto weight1 = std::make_shared<ov::op::v0::Convert>(weight1_p, ov::element::f16);
200+
auto weight2 = std::make_shared<ov::op::v0::Convert>(weight2_p, ov::element::f16);
201+
auto weight3 = std::make_shared<ov::op::v0::Convert>(weight3_p, ov::element::f16);
202+
auto bias1 = std::make_shared<ov::intel_gpu::op::Placeholder>();
203+
auto bias2 = std::make_shared<ov::intel_gpu::op::Placeholder>();
204+
auto bias3 = std::make_shared<ov::intel_gpu::op::Placeholder>();
205+
auto scale1 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{1024, 32});
206+
auto scale2 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{512, 32});
207+
auto scale3 = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{128, 32});
208+
auto fc1 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight1, bias1, scale1);
209+
auto fc2 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight2, bias2, scale2);
210+
auto fc3 = std::make_shared<ov::intel_gpu::op::FullyConnectedCompressed>(input, weight3, bias3, scale3);
211+
auto reshape_pattern = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{2}, pattern);
212+
auto reshape1 = std::make_shared<ov::op::v1::Reshape>(fc1, reshape_pattern, true);
213+
auto reshape2 = std::make_shared<ov::op::v1::Reshape>(fc2, reshape_pattern, true);
214+
auto reshape3 = std::make_shared<ov::op::v1::Reshape>(fc3, reshape_pattern, true);
215+
auto result1 = std::make_shared<ov::op::v0::Result>(reshape1);
216+
auto result2 = std::make_shared<ov::op::v0::Result>(reshape2);
217+
auto result3 = std::make_shared<ov::op::v0::Result>(reshape3);
218+
model_ref = std::make_shared<ov::Model>(ov::ResultVector{result1, result2, result3}, ov::ParameterVector{input, weight1_p, weight2_p, weight3_p});
219+
comparator.enable(FunctionsComparator::ATTRIBUTES);
220+
}
221+
}
222+
223+
TEST_F(TransformationTestsF, FullyConnectedHorizontalFusion_bias_no_zp) {
100224
std::vector<int64_t> pattern = {7, -1};
101225
{
102226
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, ov::PartialShape{-1, 7, 4096});

0 commit comments

Comments
 (0)